diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..1df343e3bb36ea14e8b6ef8b9676ec56691e2ae7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +data_folder/CyberSec-Dataset_escaped.jsonl filter=lfs diff=lfs merge=lfs -text +data_folder/Fenrir.png filter=lfs diff=lfs merge=lfs -text +kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text +kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text +kbidu_env/Lib/site-packages/accelerate/__pycache__/accelerator.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text +kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/dataclasses.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text +kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/modeling.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..1279eed7ee0805967a66b67ac249addddf1808f3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM nvidia/cuda:12.4.0-runtime-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y \ + python3.10 python3-pip git wget \ + && ln -s /usr/bin/python3.10 /usr/bin/python \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . ./KbiduAI/ +WORKDIR /workspace/KbiduAI + +ENTRYPOINT ["torchrun", "--nproc_per_node=1", "main.py"] \ No newline at end of file diff --git a/__pycache__/attention.cpython-314.pyc b/__pycache__/attention.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1950a14ec414b363171e0621fbe434e1e4a8e04c Binary files /dev/null and b/__pycache__/attention.cpython-314.pyc differ diff --git a/__pycache__/config.cpython-314.pyc b/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7dd44c1e068bcc126ac6b7b0c62882fd919a896 Binary files /dev/null and b/__pycache__/config.cpython-314.pyc differ diff --git a/__pycache__/dataset.cpython-314.pyc b/__pycache__/dataset.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13461fe702157c8ae6e2044cc8aa6d70630418f5 Binary files /dev/null and b/__pycache__/dataset.cpython-314.pyc differ diff --git a/__pycache__/embedding.cpython-314.pyc b/__pycache__/embedding.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0816dc2bd933971cdcc9505a9a632838028ec70b Binary files /dev/null and b/__pycache__/embedding.cpython-314.pyc differ diff --git a/__pycache__/feedforward.cpython-314.pyc b/__pycache__/feedforward.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0608cf480ed1692794c87f5b2c860ffdf58345f3 Binary files /dev/null and b/__pycache__/feedforward.cpython-314.pyc differ diff --git a/__pycache__/inference.cpython-314.pyc b/__pycache__/inference.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e015d0fbfc0822b7fbec4f9f88b9ec3101ccbe32 Binary files /dev/null and b/__pycache__/inference.cpython-314.pyc differ diff --git a/__pycache__/layernorm.cpython-314.pyc b/__pycache__/layernorm.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a15d26da9df4df6b1872389a5d180188b84451f Binary files /dev/null and b/__pycache__/layernorm.cpython-314.pyc differ diff --git a/__pycache__/loss.cpython-314.pyc b/__pycache__/loss.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a027b64ae94ab990eb1a8d8db15e628d9e8c054 Binary files /dev/null and b/__pycache__/loss.cpython-314.pyc differ diff --git a/__pycache__/model.cpython-314.pyc b/__pycache__/model.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..914de88d8a1dcd89fa83cd6bdc8c023c3ed15d78 Binary files /dev/null and b/__pycache__/model.cpython-314.pyc differ diff --git a/__pycache__/positional_encoding.cpython-314.pyc b/__pycache__/positional_encoding.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e08f250f51a178a2bf0281021b0c9cef54cd16c5 Binary files /dev/null and b/__pycache__/positional_encoding.cpython-314.pyc differ diff --git a/__pycache__/retriever.cpython-314.pyc b/__pycache__/retriever.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..96f296c072f623df53337512f52758989c43417c Binary files /dev/null and b/__pycache__/retriever.cpython-314.pyc differ diff --git a/__pycache__/tokenizer.cpython-314.pyc b/__pycache__/tokenizer.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13eb5b02bca223e18e0dc588339a9ed653fc1757 Binary files /dev/null and b/__pycache__/tokenizer.cpython-314.pyc differ diff --git a/__pycache__/trainer.cpython-314.pyc b/__pycache__/trainer.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b9e67c8675378e989e750a777c6a6db2d41a4e6 Binary files /dev/null and b/__pycache__/trainer.cpython-314.pyc differ diff --git a/__pycache__/transformer_block.cpython-314.pyc b/__pycache__/transformer_block.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..18aa328d04bb026a2fe0521e172a21d6d8253efd Binary files /dev/null and b/__pycache__/transformer_block.cpython-314.pyc differ diff --git a/__pycache__/utils.cpython-314.pyc b/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d2503deb24072d3862a68ab01f4260417114064 Binary files /dev/null and b/__pycache__/utils.cpython-314.pyc differ diff --git a/attention.py b/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..7c306fa507417bc7d12308f78e3592526e1127f3 --- /dev/null +++ b/attention.py @@ -0,0 +1,48 @@ +# attention.py +import torch +import torch.nn as nn +import torch.nn.functional as F + +class MultiHeadAttention(nn.Module): + def __init__(self, d_model, num_heads, num_kv_heads=None, dropout=0.1): + super().__init__() + assert d_model % num_heads == 0 + self.d_model = d_model + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads or num_heads + self.head_dim = d_model // num_heads + + self.W_q = nn.Linear(d_model, num_heads * self.head_dim, bias=False) + self.W_k = nn.Linear(d_model, self.num_kv_heads * self.head_dim, bias=False) + self.W_v = nn.Linear(d_model, self.num_kv_heads * self.head_dim, bias=False) + self.W_o = nn.Linear(num_heads * self.head_dim, d_model, bias=False) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, mask=None, rotary=None): + batch, seq_len, _ = x.size() + + q = self.W_q(x).view(batch, seq_len, self.num_heads, self.head_dim) + k = self.W_k(x).view(batch, seq_len, self.num_kv_heads, self.head_dim) + v = self.W_v(x).view(batch, seq_len, self.num_kv_heads, self.head_dim) + + if rotary is not None: + q = rotary(q, seq_len) + k = rotary(k, seq_len) + + if self.num_kv_heads != self.num_heads: + k = k.repeat_interleave(self.num_heads // self.num_kv_heads, dim=2) + v = v.repeat_interleave(self.num_heads // self.num_kv_heads, dim=2) + + # Only use is_causal if mask is None, otherwise use explicit mask + use_causal = mask is None + + attn_output = F.scaled_dot_product_attention( + q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), + attn_mask=mask, + dropout_p=self.dropout.p if self.training else 0.0, + is_causal=use_causal + ) + attn_output = attn_output.transpose(1, 2).contiguous().view(batch, seq_len, -1) + return self.W_o(attn_output) + + \ No newline at end of file diff --git a/chat_template.py b/chat_template.py new file mode 100644 index 0000000000000000000000000000000000000000..58beed46167e05c969eb0a411c6af90fb3f01f57 --- /dev/null +++ b/chat_template.py @@ -0,0 +1,36 @@ +# chat_template.py + +class ChatTemplate: + def __init__(self, system_prompt=None): + self.system_prompt = system_prompt or "You are Kbidu AI, a helpful, harmless, and honest assistant specialized in mathematics and cybersecurity." + self.history = [] + + def format_messages(self, messages): + """ + messages: list of dicts [{"role": "system/user/assistant", "content": "..."}] + Returns: formatted prompt string + """ + formatted = [] + if self.system_prompt: + formatted.append(f"System: {self.system_prompt}") + + for msg in messages: + role = msg["role"].capitalize() + content = msg["content"] + formatted.append(f"{role}: {content}") + + # Add final assistant turn + formatted.append("Assistant:") + return "\n".join(formatted) + + def build_prompt(self, query, history=None): + """Build prompt from conversation history and new query.""" + messages = [] + if history: + messages.extend(history) + messages.append({"role": "user", "content": query}) + return self.format_messages(messages) + +# Usage: +# template = ChatTemplate() +# prompt = template.build_prompt("What is RSA encryption?", history=[...]) \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000000000000000000000000000000000000..507df0aa46175bd7f6e06a55761b2d66fc38614a --- /dev/null +++ b/config.py @@ -0,0 +1,56 @@ +# config.py + +class KbiduTestConfig: + """Quick test mode – trains in ~2 minutes on a single GPU.""" + vocab_size = 1000 + d_model = 128 + num_layers = 4 + num_heads = 4 + num_kv_heads = 2 + d_ff = 256 + max_seq_len = 64 + dropout = 0.1 + layer_norm_eps = 1e-6 + use_gradient_checkpointing = True + + batch_size = 4 + learning_rate = 3e-4 + weight_decay = 0.1 + warmup_steps = 10 + total_steps = 50 + gradient_accumulation_steps = 2 + + use_streaming = True + + # --- Vision settings (for multimodal training) --- + vision_model = "openai/clip-vit-base-patch32" + freeze_vision = True + num_image_tokens = 8 + + +class Kbidu30BConfig: + """32B parameter model – production configuration.""" + vocab_size = 50000 + d_model = 7168 + num_layers = 60 + num_heads = 56 + num_kv_heads = 8 + d_ff = 19200 + max_seq_len = 2048 + dropout = 0.1 + layer_norm_eps = 1e-6 + use_gradient_checkpointing = True + + batch_size = 2 + learning_rate = 3e-4 + weight_decay = 0.1 + warmup_steps = 2000 + total_steps = 100000 + gradient_accumulation_steps = 8 + + use_streaming = True + + # --- Vision settings (for multimodal training) --- + vision_model = "openai/clip-vit-base-patch32" + freeze_vision = True + num_image_tokens = 64 \ No newline at end of file diff --git a/data/code_train.txt b/data/code_train.txt new file mode 100644 index 0000000000000000000000000000000000000000..aa180a01662b7b53eeb9dcfb4c5c1c01662b0e3b --- /dev/null +++ b/data/code_train.txt @@ -0,0 +1,276 @@ +def fibonacci(n): + if n <= 1: + return n + return fibonacci(n-1) + fibonacci(n-2) + +class Person: + def __init__(self, name, age): + self.name = name + self.age = age + + def greet(self): + return f"Hello, my name is {self.name}" + +def is_prime(n): + if n < 2: + return False + for i in range(2, int(n**0.5) + 1): + if n % i == 0: + return False + return True + +import pandas as pd +df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]}) +print(df.mean()) + +def quick_sort(arr): + if len(arr) <= 1: + return arr + pivot = arr[len(arr) // 2] + left = [x for x in arr if x < pivot] + middle = [x for x in arr if x == pivot] + right = [x for x in arr if x > pivot] + return quick_sort(left) + middle + quick_sort(right) + +class Stack: + def __init__(self): + self.items = [] + def push(self, item): + self.items.append(item) + def pop(self): + return self.items.pop() if self.items else None + def peek(self): + return self.items[-1] if self.items else None + +import requests +response = requests.get("https://api.github.com") +data = response.json() + +def read_file(filepath): + with open(filepath, 'r') as f: + return f.read() + +def write_file(filepath, content): + with open(filepath, 'w') as f: + f.write(content) + +class LinkedList: + def __init__(self): + self.head = None + def append(self, data): + new_node = Node(data) + if not self.head: + self.head = new_node + return + current = self.head + while current.next: + current = current.next + current.next = new_node + +import numpy as np +arr = np.array([1, 2, 3, 4, 5]) +print(arr.mean()) + +def binary_search(arr, target): + low, high = 0, len(arr) - 1 + while low <= high: + mid = (low + high) // 2 + if arr[mid] == target: + return mid + elif arr[mid] < target: + low = mid + 1 + else: + high = mid - 1 + return -1 + +def merge_sort(arr): + if len(arr) <= 1: + return arr + mid = len(arr) // 2 + left = merge_sort(arr[:mid]) + right = merge_sort(arr[mid:]) + return merge(left, right) + +def merge(left, right): + result = [] + i = j = 0 + while i < len(left) and j < len(right): + if left[i] <= right[j]: + result.append(left[i]) + i += 1 + else: + result.append(right[j]) + j += 1 + result.extend(left[i:]) + result.extend(right[j:]) + return result + +class Node: + def __init__(self, data): + self.data = data + self.next = None + +import re +pattern = re.compile(r'\d+') +matches = pattern.findall("There are 123 apples and 456 oranges") + +def factorial(n): + if n <= 1: + return 1 + return n * factorial(n - 1) + +def decorator(func): + def wrapper(*args, **kwargs): + print("Before function call") + result = func(*args, **kwargs) + print("After function call") + return result + return wrapper + +@decorator +def hello(): + print("Hello, World!") + +import json +data = {"name": "Alice", "age": 30} +json_str = json.dumps(data) +parsed = json.loads(json_str) + +def count_words(text): + words = text.split() + return len(words) + +class Queue: + def __init__(self): + self.items = [] + def enqueue(self, item): + self.items.append(item) + def dequeue(self): + return self.items.pop(0) if self.items else None + +def reverse_string(s): + return s[::-1] + +def is_palindrome(s): + return s == s[::-1] + +import sys +print(sys.version) + +def map_function(func, arr): + return [func(x) for x in arr] + +def filter_function(func, arr): + return [x for x in arr if func(x)] + +def reduce_function(func, arr): + result = arr[0] + for x in arr[1:]: + result = func(result, x) + return result + +import os +files = os.listdir(".") + +def power(base, exp): + return base ** exp + +def gcd(a, b): + while b: + a, b = b, a % b + return a + +def lcm(a, b): + return a * b // gcd(a, b) + +def is_even(n): + return n % 2 == 0 + +def is_odd(n): + return n % 2 != 0 + +def sum_of_squares(arr): + return sum(x**2 for x in arr) + +def average(arr): + return sum(arr) / len(arr) if arr else 0 + +def median(arr): + sorted_arr = sorted(arr) + n = len(sorted_arr) + if n % 2 == 0: + return (sorted_arr[n//2 - 1] + sorted_arr[n//2]) / 2 + return sorted_arr[n//2] + +def mode(arr): + from collections import Counter + counter = Counter(arr) + return max(counter, key=counter.get) + +def std_dev(arr): + mean = average(arr) + variance = sum((x - mean) ** 2 for x in arr) / len(arr) + return variance ** 0.5 + +import math +print(math.pi) +print(math.e) + +def sqrt(n): + return n ** 0.5 + +def log(n): + import math + return math.log(n) + +def exp(n): + import math + return math.exp(n) + +import datetime +now = datetime.datetime.now() +print(now.strftime("%Y-%m-%d %H:%M:%S")) + +def get_date(): + import datetime + return datetime.date.today() + +def get_time(): + import datetime + return datetime.datetime.now().time() + +class BankAccount: + def __init__(self, balance=0): + self.balance = balance + def deposit(self, amount): + self.balance += amount + def withdraw(self, amount): + if amount > self.balance: + raise ValueError("Insufficient funds") + self.balance -= amount + def get_balance(self): + return self.balance + +class SavingsAccount(BankAccount): + def __init__(self, balance=0, interest_rate=0.01): + super().__init__(balance) + self.interest_rate = interest_rate + def add_interest(self): + self.balance *= (1 + self.interest_rate) + +import random +print(random.randint(1, 100)) +print(random.choice(["apple", "banana", "cherry"])) + +def shuffle_list(arr): + import random + random.shuffle(arr) + return arr + +def sample_list(arr, k): + import random + return random.sample(arr, k) + +def random_choice(arr): + import random + return random.choice(arr) diff --git a/data/code_valid.txt b/data/code_valid.txt new file mode 100644 index 0000000000000000000000000000000000000000..b816ed75ca5a94d19923a8e2df91a7881d5b90df --- /dev/null +++ b/data/code_valid.txt @@ -0,0 +1,73 @@ +def quicksort(arr): + if len(arr) <= 1: + return arr + pivot = arr[len(arr)//2] + left = [x for x in arr if x < pivot] + middle = [x for x in arr if x == pivot] + right = [x for x in arr if x > pivot] + return quicksort(left) + middle + quicksort(right) + +class Animal: + def __init__(self, name): + self.name = name + def speak(self): + return f"{self.name} makes a sound" + +class Dog(Animal): + def speak(self): + return f"{self.name} barks" + +class Cat(Animal): + def speak(self): + return f"{self.name} meows" + +def parse_json(json_str): + import json + return json.loads(json_str) + +def to_json(obj): + import json + return json.dumps(obj) + +def read_csv(filepath): + import csv + with open(filepath, 'r') as f: + reader = csv.reader(f) + return list(reader) + +def write_csv(filepath, data): + import csv + with open(filepath, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerows(data) + +class ListNode: + def __init__(self, val=0, next=None): + self.val = val + self.next = next + +def reverse_linked_list(head): + prev = None + current = head + while current: + next_node = current.next + current.next = prev + prev = current + current = next_node + return prev + +def detect_cycle(head): + slow = fast = head + while fast and fast.next: + slow = slow.next + fast = fast.next.next + if slow == fast: + return True + return False + +def find_middle(head): + slow = fast = head + while fast and fast.next: + slow = slow.next + fast = fast.next.next + return slow diff --git a/data/cyber_train.txt b/data/cyber_train.txt new file mode 100644 index 0000000000000000000000000000000000000000..83d735579433d956bbb5afaeb1ee49f8076fddcc --- /dev/null +++ b/data/cyber_train.txt @@ -0,0 +1,83 @@ +SQL injection is a code injection technique that attackers use to exploit vulnerabilities in web applications. It occurs when user input is not properly sanitized and is directly embedded into SQL queries. + +Buffer overflow is a vulnerability that occurs when a program writes more data to a buffer than it can hold, causing the excess data to overflow into adjacent memory locations. + +Cross-Site Scripting (XSS) is a security vulnerability that allows attackers to inject malicious scripts into web pages viewed by other users. + +Ransomware is a type of malware that encrypts a victim's files and demands a ransom payment in exchange for the decryption key. + +A zero-day vulnerability is a previously unknown security flaw that has not yet been patched by the software vendor. Attackers can exploit these vulnerabilities before the vendor releases a fix. + +Phishing is a social engineering attack that attempts to trick victims into revealing sensitive information by masquerading as a trustworthy entity. + +The CIA triad (Confidentiality, Integrity, Availability) is the foundation of information security. + +Public-key cryptography uses a pair of keys: a public key for encryption and a private key for decryption. The public key can be shared openly, while the private key must be kept secret. + +Symmetric encryption uses the same key for both encryption and decryption. Both the sender and receiver must share the same secret key. + +A firewall is a network security device that monitors and filters incoming and outgoing network traffic based on predetermined security rules. + +Multi-factor authentication (MFA) requires users to provide two or more verification factors to gain access to a resource, significantly improving security. + +A Denial-of-Service (DoS) attack aims to make a machine or network resource unavailable by overwhelming it with a flood of illegitimate requests. + +The MITRE ATT&CK framework is a globally accessible knowledge base of adversary tactics and techniques based on real-world observations. + +Vulnerability management is the cyclical practice of identifying, classifying, remediating, and mitigating vulnerabilities. + +Penetration testing is an authorized simulated cyberattack on a computer system performed to evaluate the security of the system. + +Security Information and Event Management (SIEM) provides real-time analysis of security alerts generated by applications and network hardware. + +The OWASP Top 10 is a standard awareness document for developers and web application security, representing a broad consensus about the most critical security risks to web applications. + +Cryptography is the practice and study of techniques for secure communication in the presence of adversarial behavior. + +A vulnerability scanner is a tool that automatically identifies security weaknesses in systems and applications. + +Social engineering is the psychological manipulation of people to perform actions or divulge confidential information. + +Digital forensics is the process of uncovering and interpreting electronic data for use in a court of law. + +Endpoint Detection and Response (EDR) is a cybersecurity technology that continuously monitors and responds to advanced threats. + +Zero Trust security model assumes that no user, device, or network traffic should be trusted by default, even if it originates from inside the organization's network perimeter. + +Access control is the selective restriction of access to a resource, ensuring that only authorized users can access sensitive data. + +Data encryption transforms readable data into an unreadable format using an algorithm and a key, ensuring that only authorized parties can decrypt and access the data. + +Intrusion Detection System (IDS) monitors network traffic for suspicious activity and alerts administrators when potential threats are detected. + +Intrusion Prevention System (IPS) actively blocks detected threats in real-time, preventing them from reaching the target. + +Role-Based Access Control (RBAC) restricts system access to authorized users based on their roles within an organization. + +A man-in-the-middle attack occurs when an attacker secretly intercepts and relays messages between two parties who believe they are communicating directly with each other. + +Vulnerability exploitation is the process of taking advantage of a vulnerability to compromise a system. + +Patch management is the process of distributing and applying updates to software to fix security vulnerabilities and other bugs. + +Security audits evaluate an organization's security posture by assessing its policies, procedures, and technical controls against established standards. + +The Principle of Least Privilege states that users should be given only the minimum levels of access necessary to perform their job functions. + +Security awareness training educates employees about security risks and best practices to reduce human error and improve overall security posture. + +Ransomware attacks often use phishing emails to deliver the malware, tricking users into opening malicious attachments or clicking on malicious links. + +Two-factor authentication (2FA) is a security process that requires users to provide two different authentication factors to verify their identity. + +Security Operations Center (SOC) is a centralized unit that deals with security issues on an organizational and technical level. + +Threat intelligence is evidence-based knowledge about existing or emerging threats that can be used to inform security decisions. + +A security policy is a document that outlines an organization's security requirements, controls, and procedures. + +Breach and Attack Simulation (BAS) is a security testing methodology that continuously simulates attacks on an organization's infrastructure to identify vulnerabilities. + +Container security focuses on securing containerized applications and the infrastructure that supports them, including vulnerabilities in container images and runtime environments. + +Security orchestration, automation, and response (SOAR) is a suite of software solutions that allow organizations to collect data about security threats and respond to events without human assistance. diff --git a/data/cyber_valid.txt b/data/cyber_valid.txt new file mode 100644 index 0000000000000000000000000000000000000000..b7a18719a15c58a9f133fc5aad7ff509b0ed81be --- /dev/null +++ b/data/cyber_valid.txt @@ -0,0 +1,19 @@ +The OWASP Top 10 includes Injection, Broken Authentication, Sensitive Data Exposure, and other critical security risks. + +Defense in depth is a security strategy that layers multiple defensive mechanisms to protect systems and data, so if one layer fails, others still provide protection. + +Public Key Infrastructure (PKI) is a system that creates, manages, distributes, and revokes digital certificates. + +A certificate authority (CA) is a trusted entity that issues digital certificates used to verify the identity of individuals or organizations. + +Digital signatures provide authentication, integrity, and non-repudiation for digital documents and transactions. + +Hashing algorithms like SHA-256 are used to verify data integrity by producing a unique fixed-size hash for any given input. + +Secure Sockets Layer (SSL) and Transport Layer Security (TLS) are cryptographic protocols that provide secure communication over a computer network. + +The principle of least privilege ensures that users only have the access rights necessary to perform their job functions. + +Identity and Access Management (IAM) is the framework of policies and technologies that ensure the right individuals access the right resources at the right times for the right reasons. + +Privileged Access Management (PAM) focuses on securing, controlling, and monitoring access to privileged accounts. diff --git a/data/general_train.txt b/data/general_train.txt new file mode 100644 index 0000000000000000000000000000000000000000..259f7beefcfd60f43183eff93a541ce699172e70 --- /dev/null +++ b/data/general_train.txt @@ -0,0 +1,65 @@ +Artificial intelligence is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction. + +Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed. + +Deep learning is a type of machine learning that uses neural networks with many layers to analyze various factors of data. + +Natural language processing (NLP) is a branch of artificial intelligence that gives computers the ability to understand text and spoken words in much the same way human beings can. + +Computer vision is a field of artificial intelligence that trains computers to interpret and understand the visual world. + +The Internet of Things (IoT) refers to the network of physical objects embedded with sensors, software, and other technologies for the purpose of connecting and exchanging data with other devices and systems over the internet. + +Blockchain technology is a decentralized, distributed ledger that records the provenance of a digital asset. + +Cloud computing is the on-demand availability of computer system resources, especially data storage and computing power, without direct active management by the user. + +Quantum computing is a type of computation that harnesses the collective properties of quantum states, such as superposition, interference, and entanglement, to perform calculations. + +Big data refers to extremely large data sets that may be analyzed computationally to reveal patterns, trends, and associations. + +Data science is an interdisciplinary field that uses scientific methods, processes, algorithms, and systems to extract knowledge and insights from structured and unstructured data. + +Reinforcement learning is an area of machine learning concerned with how software agents ought to take actions in an environment to maximize the cumulative reward. + +Generative AI refers to artificial intelligence systems that can generate new content such as text, images, audio, and video. + +Neural networks are computing systems inspired by the biological neural networks that constitute animal brains. + +Large Language Models (LLMs) are a type of artificial intelligence model that uses deep learning techniques and massive data sets to understand, summarize, generate, and predict new content. + +Transformer architecture is a neural network architecture that uses self-attention mechanisms to process sequential data, and is the foundation of modern LLMs. + +The attention mechanism in deep learning allows models to focus on relevant parts of the input when generating output, significantly improving performance on tasks like translation and summarization. + +Transfer learning is a machine learning technique where a model trained on one task is reused as the starting point for a model on a second task. + +Fine-tuning is the process of taking a pre-trained model and training it further on a new dataset to adapt it to a specific task. + +Overfitting occurs when a machine learning model learns the training data too well, including its noise and outliers, resulting in poor performance on new data. + +Regularization is a technique used to prevent overfitting by adding a penalty term to the loss function to discourage complex models. + +Gradient descent is an optimization algorithm used to minimize the loss function by iteratively moving in the direction of steepest descent. + +Backpropagation is an algorithm used to train neural networks by calculating gradients of the loss function with respect to the weights. + +The learning rate is a hyperparameter that controls how much to change the model weights in response to the estimated error each time the model weights are updated. + +Batch size is the number of training examples utilized in one iteration of model training. + +Epoch refers to one complete pass through the entire training dataset during model training. + +Activation functions such as ReLU, sigmoid, and tanh introduce non-linearity into neural networks, enabling them to learn complex patterns. + +Dropout is a regularization technique where randomly selected neurons are ignored during training to prevent overfitting. + +Convolutional Neural Networks (CNNs) are a class of deep neural networks commonly applied to analyzing visual imagery. + +Recurrent Neural Networks (RNNs) are a class of neural networks that are designed to handle sequential data, such as time series or natural language. + +Attention mechanisms in neural networks allow the model to weigh the importance of different input elements, improving performance on tasks like machine translation and text summarization. + +Word embeddings are dense vector representations of words that capture semantic meaning, allowing models to understand word relationships. + +Tokenization is the process of breaking down text into smaller units called tokens, which are the fundamental building blocks for language models. diff --git a/data/general_valid.txt b/data/general_valid.txt new file mode 100644 index 0000000000000000000000000000000000000000..6e5bb1b55b8dd1c18cb941cf1c9e53c233bd41a3 --- /dev/null +++ b/data/general_valid.txt @@ -0,0 +1,35 @@ +Supervised learning is a type of machine learning where models are trained on labeled data, learning to map inputs to outputs. + +Unsupervised learning is a type of machine learning that uses unlabeled data to discover hidden patterns or data groupings. + +Semi-supervised learning uses both labeled and unlabeled data to train machine learning models, combining the strengths of supervised and unsupervised learning. + +Reinforcement learning involves training agents to make sequences of decisions by rewarding desirable behaviors and punishing undesirable ones. + +Natural language understanding (NLU) is a subset of NLP that focuses on machine comprehension of human language, enabling systems to grasp meaning, intent, and context. + +Natural language generation (NLG) is a subset of NLP that focuses on generating natural language text or speech from structured data. + +Sentiment analysis uses NLP to identify and extract subjective information from source materials, determining the emotional tone behind words. + +Machine translation is the task of automatically translating text from one language to another using computational methods. + +Text summarization is the process of creating a concise and coherent summary of a longer text document. + +Information retrieval is the process of obtaining relevant information from a collection of resources based on a user's query. + +Question answering systems automatically answer questions posed by humans in natural language. + +Named Entity Recognition (NER) is a subtask of NLP that locates and classifies named entities in text into predefined categories such as person names, organizations, locations, and more. + +Coreference resolution is the task of finding all expressions that refer to the same entity in a text. + +Dependency parsing is the process of analyzing the grammatical structure of a sentence to establish relationships between words. + +Part-of-speech tagging is the process of assigning a part-of-speech label (noun, verb, adjective, etc.) to each word in a text. + +Semantic similarity measures how similar two pieces of text are in terms of their meaning, which is crucial for many NLP applications. + +Text classification is the task of assigning predefined categories to text documents. + +Topic modeling is a technique for discovering abstract topics in a collection of documents. diff --git a/data/tokenizer/tokenizer.json b/data/tokenizer/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..5a99f1042916bbd109593beecd057005d0c4d813 --- /dev/null +++ b/data/tokenizer/tokenizer.json @@ -0,0 +1,169 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "post_processor": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": { + "": 0, + "": 1, + "": 2, + "": 3, + ".": 4, + "T": 5, + "a": 6, + "d": 7, + "e": 8, + "g": 9, + "h": 10, + "i": 11, + "l": 12, + "m": 13, + "n": 14, + "o": 15, + "p": 16, + "r": 17, + "s": 18, + "t": 19, + "v": 20, + "x": 21, + "»": 22, + "¿": 23, + "ï": 24, + "Ċ": 25, + "č": 26, + "Ġ": 27, + "is": 28, + "Ġt": 29, + "Th": 30, + "ex": 31, + "in": 32, + "»¿": 33, + "": 34, + "čĊ": 35, + "Ġis": 36, + "Ġ": 37, + "Ġtex": 38, + "This": 39, + "Ġtext": 40 + }, + "merges": [ + [ + "i", + "s" + ], + [ + "Ġ", + "t" + ], + [ + "T", + "h" + ], + [ + "e", + "x" + ], + [ + "i", + "n" + ], + [ + "»", + "¿" + ], + [ + "ï", + "»¿" + ], + [ + "č", + "Ċ" + ], + [ + "Ġ", + "is" + ], + [ + "Ġ", + "" + ], + [ + "Ġt", + "ex" + ], + [ + "Th", + "is" + ], + [ + "Ġtex", + "t" + ] + ] + } +} \ No newline at end of file diff --git a/data/train.txt b/data/train.txt new file mode 100644 index 0000000000000000000000000000000000000000..9993bb17b494d1f0adcc53778f8ab4205acbcd6d --- /dev/null +++ b/data/train.txt @@ -0,0 +1,56 @@ +def hello_world(): + print("Hello, World!") + +SQL injection is a code injection technique used to attack data-driven applications. + +Explain the concept of a buffer overflow vulnerability. + +class Calculator: + def add(self, a, b): + return a + b + +CVE-2024-1234 is a critical vulnerability in the Linux kernel. + +Write a Python function to check if a number is prime. + +Cybersecurity is the practice of protecting systems, networks, and programs from digital attacks. + +What is a zero-day vulnerability? + +def factorial(n): + if n <= 1: + return 1 + return n * factorial(n - 1) + +Ransomware is a type of malicious software that threatens to publish the victim's data. + +How does public-key cryptography work? + +import requests +response = requests.get("https://api.example.com/data") + +A firewall is a network security device that monitors and filters incoming and outgoing network traffic. + +Explain the difference between symmetric and asymmetric encryption. + +def binary_search(arr, target): + left, right = 0, len(arr) - 1 + while left <= right: + mid = (left + right) // 2 + if arr[mid] == target: + return mid + elif arr[mid] < target: + left = mid + 1 + else: + right = mid - 1 + return -1 + +What is phishing? + +Malware analysis is the process of understanding the behavior and purpose of a suspicious file or URL. + +Use the os module to list all files in a directory. + +Cross-site scripting (XSS) is a security vulnerability that allows attackers to inject malicious scripts into web pages. + +Explain the concept of defense in depth. diff --git a/data/valid.txt b/data/valid.txt new file mode 100644 index 0000000000000000000000000000000000000000..60cab582ab9689022fbf375585f45416849fcb0f --- /dev/null +++ b/data/valid.txt @@ -0,0 +1,29 @@ +What is a DDoS attack? + +A DDoS attack is a malicious attempt to disrupt normal traffic of a targeted server. + +Write a Python decorator that measures execution time. + +Encryption is the process of converting plaintext into ciphertext. + +What is the OWASP Top 10? + +def merge_sort(arr): + if len(arr) <= 1: + return arr + mid = len(arr) // 2 + left = merge_sort(arr[:mid]) + right = merge_sort(arr[mid:]) + return merge(left, right) + +What is a man-in-the-middle attack? + +Secure coding practices help prevent vulnerabilities in software. + +Explain the concept of least privilege. + +Hash functions are used to verify data integrity. + +What is a vulnerability scanner? + +Firewalls can be hardware-based or software-based. diff --git a/data_folder/.gitattributes b/data_folder/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..c5c925cc9d9c01df3110b7df80b72fa428ba80d0 --- /dev/null +++ b/data_folder/.gitattributes @@ -0,0 +1,60 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.lz4 filter=lfs diff=lfs merge=lfs -text +*.mds filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +# Audio files - uncompressed +*.pcm filter=lfs diff=lfs merge=lfs -text +*.sam filter=lfs diff=lfs merge=lfs -text +*.raw filter=lfs diff=lfs merge=lfs -text +# Audio files - compressed +*.aac filter=lfs diff=lfs merge=lfs -text +*.flac filter=lfs diff=lfs merge=lfs -text +*.mp3 filter=lfs diff=lfs merge=lfs -text +*.ogg filter=lfs diff=lfs merge=lfs -text +*.wav filter=lfs diff=lfs merge=lfs -text +# Image files - uncompressed +*.bmp filter=lfs diff=lfs merge=lfs -text +*.gif filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.tiff filter=lfs diff=lfs merge=lfs -text +# Image files - compressed +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.webp filter=lfs diff=lfs merge=lfs -text +# Video files - compressed +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.webm filter=lfs diff=lfs merge=lfs -text +CyberSec-Dataset_escaped.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/data_folder/CyberSec-Dataset_escaped.jsonl b/data_folder/CyberSec-Dataset_escaped.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5781a6bd5242cad381f7c1d7e158f1a4ac744397 --- /dev/null +++ b/data_folder/CyberSec-Dataset_escaped.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da1d7b888aa697bd44b6e24ea95e02277c34bcd40c9cf6e701cf72bcac0f9143 +size 360844505 diff --git a/data_folder/Fenrir.png b/data_folder/Fenrir.png new file mode 100644 index 0000000000000000000000000000000000000000..b7a81e7700da9b4baa5a323469a76ce54201c716 --- /dev/null +++ b/data_folder/Fenrir.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb5056862ee67f4e13f1ebfe422146b03d994ae249c4794d5684113674699df1 +size 1099144 diff --git a/data_folder/README.md b/data_folder/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8d9451043298a8577237e8c688e1385f8d67c17f --- /dev/null +++ b/data_folder/README.md @@ -0,0 +1,156 @@ +--- +license: apache-2.0 +task_categories: + - text-generation +language: + - en +tags: + - cybersecurity + - defensive-security + - instruction-tuning +size_categories: + - 10K + +Created by Alican Kiraz + + + +## TL;DR +A ready-to-train dataset of **83,920** high-quality *system / user / assistant* triples for **defensive, alignment-safe cybersecurity SFT** training. +Apache-2.0 licensed and production-ready. +**Scope:** OWASP Top 10, MITRE ATT&CK, NIST CSF, CIS Controls, ASD Essential 8, modern authentication (OAuth 2 / OIDC / SAML), SSL / TLS, Cloud & DevSecOps, Cryptography, and AI Security. + +--- + +## 1  What’s new in v2.0  (2025‑10‑06) + +| Change | v1.1.0 | **v2.0.0** | +| ----------------- | ------------------------------------: | ----------------------------------------------------------------------------------------------: | +| **Rows** | 21 258 | **83 920** *(≈ 4×)* | +| **Coverage** | OWASP, NIST CSF, +MITRE ATT&CK, CIS | + Deeper Cloud (AWS/Azure/GCP), modern auth hardening, crypto hygiene, AI‑security interplay | +| **Quality gates** | Dedup, PII scrub, hallucination scans | + adversarial refusal tests (jailbreak/injection), static policy linting, content risk taxonomy | +| **Format** | Parquet (chat triples) | + stricter schema checks, stable row IDs | +| **License** | Apache‑2.0 | Apache‑2.0 | + + +**Highlights** + +* **Big**: 83,920 chat samples with explanations at **senior security engineer** level. +* **Safe-by-design**: Built-in **rejection patterns** and *alignment* checks against malicious or exploit requests. +* **Framework-aware**: Content is **mapped to standards** (OWASP / ATT&CK / NIST / CIS). +* **Cloud & DevSecOps first**: IAM, secrets, CI/CD, container / k8s hardening, logging / SIEM, incident response. +* **Commercial-friendly**: **Apache-2.0** licensed. + +--- + +## 2  Dataset Summary + +| Property | Value | +| ------------ | ------------------------------------------------------ | +| **Language** | English | +| **License** | Apache‑2.0 | +| **Format** | Parquet (columnar) | +| **Rows** | **83 920** | +| **Columns** | `system`, `user`, `assistant` | +| **Split** | `train` (100 %) | + +**Record schema (chat triple)** + +```json +{ + "system": "You are a seasoned cyber‑defense AI that follows industry ethics...", + "user": "Compare mitigations for Reflected vs Stored XSS in a modern SPA.", + "assistant": "Reflected and Stored XSS share core mitigation pillars—output encoding..." +} +``` + +--- + +## 3. Coverage & Design + +### 3.1 Domains & Frameworks + +* **AppSec & Web**: OWASP Top 10, secure coding, input/output handling, SSRF, deserialization. +* **Cloud Security**: IAM guardrails, least privilege, key rotation, KMS/HSM, network segmentation, posture mgmt. +* **DevSecOps**: SAST/DAST, SBOM, supply‑chain, CI/CD signing, container & Kubernetes hardening. +* **Identity & Access**: OAuth2/OIDC/SAML, MFA/Phishing‑resistant auth, session mgmt. +* **Crypto Hygiene**: TLS configs, AEAD modes, key lifecycle, randomness, password hashing. +* **Detection & Response**: logging, SIEM correlation, threat hunting, IR playbooks. +* **AI‑Security Interplay**: prompt injection defense, data‑poisoning awareness, model‑misuse refusals. + +### 3.2 Instruction styles + +* Compare/contrast, step‑by‑step mitigation, checklists, “why it fails” root‑cause analyses, policy rationale, trade‑offs, and “refuse with explanation” for dual‑use prompts. + +--- + +## 4. Data Creation & Quality + +1. **Source harvesting**: 250 k+ public technical docs (standards, RFCs, white‑papers, vendor guidance). +2. **Extraction**: boilerplate stripping, language detection, heuristic paragraph segmentation. +3. **Topical filtering**: keyword+embedding retrieval towards defensive security only. +4. **Instruction synthesis**: prompts → *system/user/assistant*; enforced ethics & refusal templates. +5. **Quality gates** *(multi‑layer)* + + * **Deduplication**: MinHash + LSH cluster pruning. + * **PII & profanity scrub**. + * **Hallucination/inconsistency scans** (LLM‑aided). + * **Refusal‑pattern tests**: jailbreak & prompt‑injection triggers; no exploit‑building steps. + * **Manual spot review** (~3 % sample). + +--- + +## 5. Ethical Use & Safety + +* **Dual‑use risk**: Dataset intentionally avoids exploit crafting; offensive requests receive **explanatory refusals**. +* **Bias**: Focus on widely used frameworks (OWASP/NIST/CIS). + + * *Roadmap*: more regional standards (e.g., ISO/IEC, GDPR security controls). +* **Provenance**: Only public sources; licensing respected; outputs released under **Apache‑2.0**. + +--- + +## 6. Limitations + +* English‑only. +* Predominantly defensive stance; red‑team tactics only for mitigation context. +* Security evolves rapidly; periodic refresh planned. + +--- + +## 7. Example Records + +**Mitigation checklist:** hardening steps, rationales, pitfalls, references to standards. +**Refusal sample:** clearly declines malware/exploit construction with safe alternatives (logging, detection, patching). + +> *All examples adhere to the `system/user/assistant` schema and are engineered to be alignment‑safe.* + +--- + +## 8. Citation + +```bibtex +@dataset{alican_kiraz_2025_heimdall_v2_0, + author = {Alican Kiraz}, + title = {Fenrir v2.0 — Cybersecurity Defense Instruction-Tuning Dataset}, + year = {2025}, + publisher = {Hugging Face}, + url = {https://huggingface.co/datasets/AlicanKiraz0/Cybersecurity-Dataset-Heimdall-v2.0} +} +``` + +--- + +## 9. Changelog + +* **v2.0.0** (2025‑10‑06) — Expanded to **83 920** rows; deeper Cloud/DevSecOps/Identity coverage; stronger adversarial refusal tests; stricter schema checks. +* **v1.1.0** (2025‑06‑21) — 21 258 rows; broadened framework coverage; improved automatic quality gates. +* **v1.0.0** (2025‑06‑17) — Initial 2 500 rows. + +--- \ No newline at end of file diff --git a/data_preprocessor.py b/data_preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..b85d8d68ba86c8f445bfaec87b4026ae8d532890 --- /dev/null +++ b/data_preprocessor.py @@ -0,0 +1,63 @@ +# data_preprocessor.py +import hashlib +import re +from collections import Counter + +class DataPreprocessor: + def __init__(self): + self.seen_hashes = set() + + def deduplicate(self, texts): + """Remove exact duplicate lines.""" + unique_texts = [] + for text in texts: + text_hash = hashlib.md5(text.encode()).hexdigest() + if text_hash not in self.seen_hashes: + self.seen_hashes.add(text_hash) + unique_texts.append(text) + print(f"Removed {len(texts) - len(unique_texts)} duplicates.") + return unique_texts + + def filter_quality(self, texts, min_words=5, max_word_ratio=0.9): + """Filter short texts and those with too many repeated words.""" + filtered = [] + for text in texts: + words = text.split() + if len(words) < min_words: + continue + # Check if > 90% of words are the same (spam) + if len(words) > 0 and max(Counter(words).values()) / len(words) > max_word_ratio: + continue + filtered.append(text) + print(f"Removed {len(texts) - len(filtered)} low-quality texts.") + return filtered + + def remove_toxic(self, texts, toxic_keywords=None): + """Basic toxicity filter.""" + if toxic_keywords is None: + toxic_keywords = ["hate", "violence", "kill", "racist", "sexist", "nazi"] + filtered = [] + for text in texts: + lower_text = text.lower() + if any(kw in lower_text for kw in toxic_keywords): + continue + filtered.append(text) + print(f"Removed {len(texts) - len(filtered)} potentially toxic texts.") + return filtered + + def preprocess_file(self, input_path, output_path): + with open(input_path, 'r', encoding='utf-8') as f: + texts = [line.strip() for line in f if line.strip()] + + texts = self.deduplicate(texts) + texts = self.filter_quality(texts) + texts = self.remove_toxic(texts) + + with open(output_path, 'w', encoding='utf-8') as f: + for text in texts: + f.write(text + '\n') + print(f"Saved cleaned data to {output_path}") + +# Usage: +# preprocessor = DataPreprocessor() +# preprocessor.preprocess_file("data/raw_train.txt", "data/train.txt") \ No newline at end of file diff --git a/dataset.py b/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..68fc9eb8f30e266a78a419834a5fc40c6138ca85 --- /dev/null +++ b/dataset.py @@ -0,0 +1,181 @@ +# dataset.py +import torch +from torch.utils.data import Dataset, IterableDataset +import os +import random +from datasets import load_dataset as hf_load_dataset + + +class TextDataset(Dataset): + """ + Simple dataset for a single text file (used for validation). + """ + def __init__(self, file_path, tokenizer, max_len): + self.tokenizer = tokenizer + self.max_len = max_len + with open(file_path, 'r', encoding='utf-8') as f: + self.texts = [line.strip() for line in f if line.strip()] + + def __len__(self): + return len(self.texts) + + def __getitem__(self, idx): + text = self.texts[idx] + if hasattr(self.tokenizer, 'encode'): + ids = self.tokenizer.encode(text) + else: + ids = self.tokenizer.encode(text, add_special_tokens=True) + if len(ids) > self.max_len: + ids = ids[:self.max_len] + else: + if hasattr(self.tokenizer, 'pad_id'): + pad_id = self.tokenizer.pad_id + elif hasattr(self.tokenizer, 'pad_token_id'): + pad_id = self.tokenizer.pad_token_id + else: + pad_id = 0 + ids = ids + [pad_id] * (self.max_len - len(ids)) + return torch.tensor(ids, dtype=torch.long) + + +class MultiSourceTextDataset(Dataset): + """ + Flexible dataset that can load from: + - Local text files + - Hugging Face datasets + """ + def __init__(self, tokenizer, max_len=512, local_files=None, hf_datasets=None, shuffle=True): + self.tokenizer = tokenizer + self.max_len = max_len + self.shuffle = shuffle + self.data = [] + + if local_files: + for file_path in local_files: + if os.path.exists(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + lines = [line.strip() for line in f if line.strip()] + self.data.extend(lines) + print(f" Loaded {len(lines)} from {file_path}") + + if hf_datasets: + for ds_name in hf_datasets: + try: + ds = hf_load_dataset(ds_name, split="train", trust_remote_code=True) + if "content" in ds.column_names: + texts = [item["content"] for item in ds] + elif "text" in ds.column_names: + texts = [item["text"] for item in ds] + else: + col = ds.column_names[0] + texts = [item[col] for item in ds] + self.data.extend(texts) + print(f" Loaded {len(texts)} from {ds_name}") + except Exception as e: + print(f" ⚠️ Error loading {ds_name}: {e}") + + if not self.data: + print("⚠️ No data loaded!") + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + text = self.data[idx] + if hasattr(self.tokenizer, 'encode'): + ids = self.tokenizer.encode(text) + else: + ids = self.tokenizer.encode(text, add_special_tokens=True) + if len(ids) > self.max_len: + ids = ids[:self.max_len] + else: + if hasattr(self.tokenizer, 'pad_id'): + pad_id = self.tokenizer.pad_id + elif hasattr(self.tokenizer, 'pad_token_id'): + pad_id = self.tokenizer.pad_token_id + else: + pad_id = 0 + ids = ids + [pad_id] * (self.max_len - len(ids)) + return torch.tensor(ids, dtype=torch.long) + + +class StreamingMultiSourceDataset(IterableDataset): + """ + Memory-efficient streaming dataset for huge datasets. + """ + def __init__(self, tokenizer, max_len=512, file_paths=None, shuffle=True, buffer_size=10000): + self.tokenizer = tokenizer + self.max_len = max_len + self.file_paths = file_paths or [] + self.shuffle = shuffle + self.buffer_size = buffer_size + + def __iter__(self): + for file_path in self.file_paths: + if not os.path.exists(file_path): + continue + with open(file_path, 'r', encoding='utf-8') as f: + buffer = [] + for line in f: + line = line.strip() + if line: + if self.shuffle: + buffer.append(line) + if len(buffer) >= self.buffer_size: + random.shuffle(buffer) + for item in buffer: + yield self._tokenize(item) + buffer = [] + else: + yield self._tokenize(line) + if buffer: + random.shuffle(buffer) + for item in buffer: + yield self._tokenize(item) + + def _tokenize(self, text): + if hasattr(self.tokenizer, 'encode'): + ids = self.tokenizer.encode(text) + else: + ids = self.tokenizer.encode(text, add_special_tokens=True) + if len(ids) > self.max_len: + ids = ids[:self.max_len] + else: + if hasattr(self.tokenizer, 'pad_id'): + pad_id = self.tokenizer.pad_id + elif hasattr(self.tokenizer, 'pad_token_id'): + pad_id = self.tokenizer.pad_token_id + else: + pad_id = 0 + ids = ids + [pad_id] * (self.max_len - len(ids)) + return torch.tensor(ids, dtype=torch.long) + + +def create_mixed_dataset(tokenizer, max_len=512, config=None): + """ + Factory function that creates a mixed dataset. + Uses streaming if config.use_streaming is True. + """ + local_files = [] + for f in ["data/code_train.txt", "data/cyber_train.txt", "data/general_train.txt", "data/train.txt"]: + if os.path.exists(f): + local_files.append(f) + + use_streaming = config and hasattr(config, 'use_streaming') and config.use_streaming + + if use_streaming: + return StreamingMultiSourceDataset( + tokenizer=tokenizer, + max_len=max_len, + file_paths=local_files, + shuffle=True, + buffer_size=10000 + ) + else: + return MultiSourceTextDataset( + tokenizer=tokenizer, + max_len=max_len, + local_files=local_files, + hf_datasets=[], + shuffle=True + ) \ No newline at end of file diff --git a/dpo_trainer.py b/dpo_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..9724e4220b398bd2fc96dbdf6a9a06c113af1e8f --- /dev/null +++ b/dpo_trainer.py @@ -0,0 +1,99 @@ +# dpo_trainer.py +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import Dataset, DataLoader +from transformers import AutoTokenizer +import random + +class PreferenceDataset(Dataset): + def __init__(self, data_path, tokenizer, max_len=512): + self.tokenizer = tokenizer + self.max_len = max_len + self.data = [] + with open(data_path, 'r', encoding='utf-8') as f: + for line in f: + parts = line.strip().split('\t') + if len(parts) == 3: + prompt, chosen, rejected = parts + self.data.append((prompt, chosen, rejected)) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + prompt, chosen, rejected = self.data[idx] + # Encode + prompt_ids = self.tokenizer.encode(prompt) + chosen_ids = prompt_ids + self.tokenizer.encode(chosen)[1:] # remove BOS + rejected_ids = prompt_ids + self.tokenizer.encode(rejected)[1:] + # Pad + if len(chosen_ids) > self.max_len: + chosen_ids = chosen_ids[:self.max_len] + else: + chosen_ids += [self.tokenizer.pad_id] * (self.max_len - len(chosen_ids)) + if len(rejected_ids) > self.max_len: + rejected_ids = rejected_ids[:self.max_len] + else: + rejected_ids += [self.tokenizer.pad_id] * (self.max_len - len(rejected_ids)) + return torch.tensor(chosen_ids, dtype=torch.long), torch.tensor(rejected_ids, dtype=torch.long) + +class DPOTrainer: + def __init__(self, model, ref_model, train_loader, config, device, beta=0.1, lr=1e-5): + self.model = model + self.ref_model = ref_model + self.train_loader = train_loader + self.config = config + self.device = device + self.beta = beta + self.optimizer = torch.optim.AdamW(model.parameters(), lr=lr) + + def compute_dpo_loss(self, chosen_logps, rejected_logps): + # DPO loss: -log(sigmoid(beta * (chosen - rejected))) + diff = chosen_logps - rejected_logps + loss = -F.logsigmoid(self.beta * diff).mean() + return loss + + def train_step(self, batch_chosen, batch_rejected): + batch_chosen = batch_chosen.to(self.device) + batch_rejected = batch_rejected.to(self.device) + + # Forward pass on chosen + logits_chosen = self.model(batch_chosen) + logits_ref_chosen = self.ref_model(batch_chosen) + + # Forward pass on rejected + logits_rejected = self.model(batch_rejected) + logits_ref_rejected = self.ref_model(batch_rejected) + + # Compute log probabilities (ignoring padding) + logps_chosen = F.log_softmax(logits_chosen, dim=-1).gather(-1, batch_chosen.unsqueeze(-1)).squeeze(-1).sum(-1) + logps_ref_chosen = F.log_softmax(logits_ref_chosen, dim=-1).gather(-1, batch_chosen.unsqueeze(-1)).squeeze(-1).sum(-1) + + logps_rejected = F.log_softmax(logits_rejected, dim=-1).gather(-1, batch_rejected.unsqueeze(-1)).squeeze(-1).sum(-1) + logps_ref_rejected = F.log_softmax(logits_ref_rejected, dim=-1).gather(-1, batch_rejected.unsqueeze(-1)).squeeze(-1).sum(-1) + + # DPO loss + loss = self.compute_dpo_loss(logps_chosen - logps_ref_chosen, logps_rejected - logps_ref_rejected) + + self.optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0) + self.optimizer.step() + return loss.item() + + def train(self, epochs=1): + self.model.train() + for epoch in range(epochs): + total_loss = 0 + for batch_chosen, batch_rejected in self.train_loader: + loss = self.train_step(batch_chosen, batch_rejected) + total_loss += loss + print(f"DPO Epoch {epoch+1}, Loss: {total_loss/len(self.train_loader):.4f}") + +# Usage: +# dpo_dataset = PreferenceDataset("data/preference.txt", tokenizer) +# dpo_loader = DataLoader(dpo_dataset, batch_size=4, shuffle=True) +# ref_model = KbiduForCausalLM(config) # copy of base model (frozen) +# dpo_trainer = DPOTrainer(model, ref_model, dpo_loader, config, device) +# dpo_trainer.train(epochs=3) \ No newline at end of file diff --git a/embedding.py b/embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..4cb25082ea2550d674c81e196e843bb1b0ecc58c --- /dev/null +++ b/embedding.py @@ -0,0 +1,11 @@ +# embedding.py +import torch.nn as nn + +class Embedding(nn.Module): + def __init__(self, vocab_size, d_model): + super().__init__() + self.d_model = d_model + self.embed = nn.Embedding(vocab_size, d_model) + + def forward(self, x): + return self.embed(x) * (self.d_model ** 0.5) diff --git a/eval_math.py b/eval_math.py new file mode 100644 index 0000000000000000000000000000000000000000..68be50274507db6ff4dde805fcfe832a5d617fe8 --- /dev/null +++ b/eval_math.py @@ -0,0 +1,48 @@ +# eval_math.py +import re +from datasets import load_dataset +from inference import Inference +from tqdm import tqdm + +def extract_answer(text): + """Extract final numeric answer from text.""" + # Try to find the last number in the text + numbers = re.findall(r'[-+]?\d*\.?\d+', text) + if numbers: + return numbers[-1] + return None + +def evaluate_gsm8k(model, tokenizer, device, num_samples=100): + """Evaluate on GSM8K dataset.""" + dataset = load_dataset("gsm8k", "main", split="test") + total = 0 + correct = 0 + inference_engine = Inference(model, tokenizer, device, max_new_tokens=200) + + for i, example in enumerate(tqdm(dataset.select(range(num_samples)))): + question = example["question"] + true_answer = example["answer"].split("####")[-1].strip() + + # Generate with CoT + output = inference_engine.generate(question, use_cot=True) + pred_answer = extract_answer(output) + + if pred_answer and pred_answer in true_answer: + correct += 1 + total += 1 + + accuracy = correct / total + print(f"GSM8K Accuracy ({num_samples} samples): {accuracy:.2%}") + return accuracy + +def evaluate_math(model, tokenizer, device, num_samples=100): + """Evaluate on MATH dataset (college math).""" + # Similar logic for MATH dataset + dataset = load_dataset("math_dataset", split="test") + # ... implementation similar to above + pass + +# Usage: +# from model import KbiduForCausalLM +# model = KbiduForCausalLM(config).to(device) +# evaluate_gsm8k(model, tokenizer, device, num_samples=50) \ No newline at end of file diff --git a/faiss_index.faiss b/faiss_index.faiss new file mode 100644 index 0000000000000000000000000000000000000000..d655c9455a8e0ceda14c0a3aa4077ddb0a1cc402 Binary files /dev/null and b/faiss_index.faiss differ diff --git a/faiss_index_chunks.txt b/faiss_index_chunks.txt new file mode 100644 index 0000000000000000000000000000000000000000..e02330b17dc66c7e6e46656575345defc08c78a0 --- /dev/null +++ b/faiss_index_chunks.txt @@ -0,0 +1 @@ +This is sample training text. diff --git a/feedforward.py b/feedforward.py new file mode 100644 index 0000000000000000000000000000000000000000..9538ff09b57b389b51b41e959ea413b904098ea3 --- /dev/null +++ b/feedforward.py @@ -0,0 +1,14 @@ +# feedforward.py +import torch.nn as nn +import torch.nn.functional as F + +class SwiGLU(nn.Module): + def __init__(self, d_model, d_ff, dropout=0.1): + super().__init__() + self.w1 = nn.Linear(d_model, d_ff, bias=False) + self.w2 = nn.Linear(d_ff, d_model, bias=False) + self.w3 = nn.Linear(d_model, d_ff, bias=False) + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + return self.w2(self.dropout(F.silu(self.w1(x)) * self.w3(x))) diff --git a/inference.py b/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..0d36db637f9d861e235cce4a3a34f7b88269e621 --- /dev/null +++ b/inference.py @@ -0,0 +1,107 @@ +# inference.py +import torch +import torch.nn.functional as F +from collections import Counter +import re + +class Inference: + def __init__(self, model, tokenizer, device, retriever=None, max_new_tokens=300, memory_size=5): + self.model = model + self.tokenizer = tokenizer + self.device = device + self.retriever = retriever + self.max_new_tokens = max_new_tokens + self.memory = [] + self.memory_size = memory_size + + def _build_prompt(self, query, use_cot=True, tone="empathetic", think_before_answer=False): + tone_map = { + "empathetic": "You are Kbidu AI, a warm, kind, and emotionally intelligent assistant.", + "professional": "You are Kbidu AI, a precise, clear, and objective assistant.", + "humorous": "You are Kbidu AI, a witty and fun assistant.", + "supportive": "You are Kbidu AI, a supportive and encouraging mentor." + } + system_prompt = tone_map.get(tone, tone_map["empathetic"]) + + # RAG context + context = "" + if self.retriever: + results = self.retriever.retrieve(query, top_k=3) + if results: + context = "Relevant information:\n" + "\n".join([r["chunk"] for r in results]) + "\n\n" + + # Conversation memory + memory_text = "" + if self.memory: + memory_text = "Conversation history:\n" + for user, assistant in self.memory[-self.memory_size:]: + memory_text += f"User: {user}\nAssistant: {assistant}\n" + memory_text += "\n" + + # Think-before-answer instruction + think_instruction = "" + if think_before_answer: + think_instruction = ( + "Before giving your final answer, think step by step inside your head.\n" + "Then, provide your final answer after the line 'Final Answer:'.\n" + "Do not include your reasoning in the final output unless asked.\n" + ) + + cot_instruction = "Let's think step by step.\n" if use_cot else "" + + prompt = f"{system_prompt}\n\n{context}{memory_text}{think_instruction}{cot_instruction}User: {query}\nAssistant:" + return prompt + + def generate(self, query, use_cot=True, temperature=0.7, tone="empathetic", think_before_answer=False, show_reasoning=False): + prompt = self._build_prompt(query, use_cot, tone, think_before_answer) + input_ids = torch.tensor([self.tokenizer.encode(prompt)]).to(self.device) + + for _ in range(self.max_new_tokens): + with torch.no_grad(): + logits = self.model(input_ids) + next_logits = logits[0, -1, :] / temperature + probs = F.softmax(next_logits, dim=-1) + next_token = torch.multinomial(probs, num_samples=1).unsqueeze(0) + input_ids = torch.cat([input_ids, next_token], dim=-1) + if next_token.item() == self.tokenizer.eos_id: + break + + full_response = self.tokenizer.decode(input_ids[0].tolist()) + + # Extract assistant part + if "Assistant:" in full_response: + response = full_response.split("Assistant:", 1)[-1].strip() + else: + response = full_response[len(prompt):].strip() + + # If think_before_answer is True, try to extract final answer after "Final Answer:" + if think_before_answer: + if "Final Answer:" in response: + final_answer = response.split("Final Answer:", 1)[-1].strip() + if show_reasoning: + # Return the full response (reasoning + final answer) + self.memory.append((query, response)) + return response + else: + # Return only the final answer + self.memory.append((query, final_answer)) + return final_answer + else: + # If no "Final Answer:" found, return the full response (fallback) + self.memory.append((query, response)) + return response + else: + self.memory.append((query, response)) + return response + + def generate_with_self_consistency(self, query, n=5, temperature=0.7, tone="empathetic", think_before_answer=False): + answers = [] + for _ in range(n): + ans = self.generate(query, use_cot=True, temperature=temperature, tone=tone, think_before_answer=think_before_answer, show_reasoning=False) + if "Answer:" in ans: + ans = ans.split("Answer:")[-1].strip() + answers.append(ans) + if answers: + counter = Counter(answers) + return counter.most_common(1)[0][0] + return answers[0] if answers else "" \ No newline at end of file diff --git a/kbidu_env/.gitignore b/kbidu_env/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..49b53dcfd66bcbbee1f9f854a995842a8263d3d3 --- /dev/null +++ b/kbidu_env/.gitignore @@ -0,0 +1,2 @@ +# Created by venv; see https://docs.python.org/3/library/venv.html +* diff --git a/kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd b/kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..fb6cb07b9f650a8a93bcb20ae6b63d78edd0d73b --- /dev/null +++ b/kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab64835b63093c31975df2692756c8008e7ec190e4aa86f15a713bcc62e1d432 +size 216576 diff --git a/kbidu_env/Lib/site-packages/__pycache__/isympy.cpython-314.pyc b/kbidu_env/Lib/site-packages/__pycache__/isympy.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7fda34dc975368cc884c4df564720f871321b601 Binary files /dev/null and b/kbidu_env/Lib/site-packages/__pycache__/isympy.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/__pycache__/six.cpython-314.pyc b/kbidu_env/Lib/site-packages/__pycache__/six.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..810921ce226c8409db088bfac225f8dc7cac2766 Binary files /dev/null and b/kbidu_env/Lib/site-packages/__pycache__/six.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/__pycache__/threadpoolctl.cpython-314.pyc b/kbidu_env/Lib/site-packages/__pycache__/threadpoolctl.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4446caf679a04e967cd7bd5a8c23f3863d902c2 Binary files /dev/null and b/kbidu_env/Lib/site-packages/__pycache__/threadpoolctl.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc b/kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5629b0c56307bd2159fbdf0d55219064d2516024 --- /dev/null +++ b/kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42c2b14d3bd1c9fc087ac29ced4cef30dc3b08c3a322502df232532b1cb6aa1f +size 175506 diff --git a/kbidu_env/Lib/site-packages/_distutils_hack/__init__.py b/kbidu_env/Lib/site-packages/_distutils_hack/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4d3f09b0ae97c9d614522eb9a44533b32543cb9b --- /dev/null +++ b/kbidu_env/Lib/site-packages/_distutils_hack/__init__.py @@ -0,0 +1,220 @@ +# don't import any costly modules +import sys +import os + + +def warn_distutils_present(): + if 'distutils' not in sys.modules: + return + import warnings + + warnings.warn( + "Distutils was imported before Setuptools, but importing Setuptools " + "also replaces the `distutils` module in `sys.modules`. This may lead " + "to undesirable behaviors or errors. To avoid these issues, avoid " + "using distutils directly, ensure that setuptools is installed in the " + "traditional way (e.g. not an editable install), and/or make sure " + "that setuptools is always imported before distutils." + ) + + +def clear_distutils(): + if 'distutils' not in sys.modules: + return + import warnings + + warnings.warn("Setuptools is replacing distutils.") + mods = [ + name + for name in sys.modules + if name == "distutils" or name.startswith("distutils.") + ] + for name in mods: + del sys.modules[name] + + +def enabled(): + """ + Allow selection of distutils by environment variable. + """ + which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local') + return which == 'local' + + +def ensure_local_distutils(): + import importlib + + clear_distutils() + + # With the DistutilsMetaFinder in place, + # perform an import to cause distutils to be + # loaded from setuptools._distutils. Ref #2906. + with shim(): + importlib.import_module('distutils') + + # check that submodules load as expected + core = importlib.import_module('distutils.core') + assert '_distutils' in core.__file__, core.__file__ + assert 'setuptools._distutils.log' not in sys.modules + + +def do_override(): + """ + Ensure that the local copy of distutils is preferred over stdlib. + + See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 + for more motivation. + """ + if enabled(): + warn_distutils_present() + ensure_local_distutils() + + +class _TrivialRe: + def __init__(self, *patterns): + self._patterns = patterns + + def match(self, string): + return all(pat in string for pat in self._patterns) + + +class DistutilsMetaFinder: + def find_spec(self, fullname, path, target=None): + # optimization: only consider top level modules and those + # found in the CPython test suite. + if path is not None and not fullname.startswith('test.'): + return None + + method_name = 'spec_for_{fullname}'.format(**locals()) + method = getattr(self, method_name, lambda: None) + return method() + + def spec_for_distutils(self): + if self.is_cpython(): + return None + + import importlib + import importlib.abc + import importlib.util + + try: + mod = importlib.import_module('setuptools._distutils') + except Exception: + # There are a couple of cases where setuptools._distutils + # may not be present: + # - An older Setuptools without a local distutils is + # taking precedence. Ref #2957. + # - Path manipulation during sitecustomize removes + # setuptools from the path but only after the hook + # has been loaded. Ref #2980. + # In either case, fall back to stdlib behavior. + return None + + class DistutilsLoader(importlib.abc.Loader): + def create_module(self, spec): + mod.__name__ = 'distutils' + return mod + + def exec_module(self, module): + pass + + return importlib.util.spec_from_loader( + 'distutils', DistutilsLoader(), origin=mod.__file__ + ) + + @staticmethod + def is_cpython(): + """ + Suppress supplying distutils for CPython (build and tests). + Ref #2965 and #3007. + """ + return os.path.isfile('pybuilddir.txt') + + def spec_for_pip(self): + """ + Ensure stdlib distutils when running under pip. + See pypa/pip#8761 for rationale. + """ + if sys.version_info >= (3, 12) or self.pip_imported_during_build(): + return + clear_distutils() + self.spec_for_distutils = lambda: None + + @classmethod + def pip_imported_during_build(cls): + """ + Detect if pip is being imported in a build script. Ref #2355. + """ + import traceback + + return any( + cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None) + ) + + @staticmethod + def frame_file_is_setup(frame): + """ + Return True if the indicated frame suggests a setup.py file. + """ + # some frames may not have __file__ (#2940) + return frame.f_globals.get('__file__', '').endswith('setup.py') + + def spec_for_sensitive_tests(self): + """ + Ensure stdlib distutils when running select tests under CPython. + + python/cpython#91169 + """ + clear_distutils() + self.spec_for_distutils = lambda: None + + sensitive_tests = ( + [ + 'test.test_distutils', + 'test.test_peg_generator', + 'test.test_importlib', + ] + if sys.version_info < (3, 10) + else [ + 'test.test_distutils', + ] + ) + + +for name in DistutilsMetaFinder.sensitive_tests: + setattr( + DistutilsMetaFinder, + f'spec_for_{name}', + DistutilsMetaFinder.spec_for_sensitive_tests, + ) + + +DISTUTILS_FINDER = DistutilsMetaFinder() + + +def add_shim(): + DISTUTILS_FINDER in sys.meta_path or insert_shim() + + +class shim: + def __enter__(self): + insert_shim() + + def __exit__(self, exc, value, tb): + _remove_shim() + + +def insert_shim(): + sys.meta_path.insert(0, DISTUTILS_FINDER) + + +def _remove_shim(): + try: + sys.meta_path.remove(DISTUTILS_FINDER) + except ValueError: + pass + + +if sys.version_info < (3, 12): + # DistutilsMetaFinder can only be disabled in Python < 3.12 (PEP 632) + remove_shim = _remove_shim diff --git a/kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d446bae13511ac5a0eff5a1957edad9dc0c2b0c Binary files /dev/null and b/kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/override.cpython-314.pyc b/kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/override.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08ef99fa37fb186a8615eb96e907b001198f89ea Binary files /dev/null and b/kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/override.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/_distutils_hack/override.py b/kbidu_env/Lib/site-packages/_distutils_hack/override.py new file mode 100644 index 0000000000000000000000000000000000000000..2cc433a4a55e3b41fa31089918fb62096092f89f --- /dev/null +++ b/kbidu_env/Lib/site-packages/_distutils_hack/override.py @@ -0,0 +1 @@ +__import__('_distutils_hack').do_override() diff --git a/kbidu_env/Lib/site-packages/_multiprocess/__init__.py b/kbidu_env/Lib/site-packages/_multiprocess/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9a89f71736472fe74c8f3a5ffac26b9bb8dcc9a2 --- /dev/null +++ b/kbidu_env/Lib/site-packages/_multiprocess/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2022-2026 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/multiprocess/blob/master/LICENSE + +from _multiprocessing import * diff --git a/kbidu_env/Lib/site-packages/_multiprocess/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/_multiprocess/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24a2cf41cea17b0668d967e283632d0e8922aed2 Binary files /dev/null and b/kbidu_env/Lib/site-packages/_multiprocess/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/_yaml/__init__.py b/kbidu_env/Lib/site-packages/_yaml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7baa8c4b68127d5cdf0be9a799429e61347c2694 --- /dev/null +++ b/kbidu_env/Lib/site-packages/_yaml/__init__.py @@ -0,0 +1,33 @@ +# This is a stub package designed to roughly emulate the _yaml +# extension module, which previously existed as a standalone module +# and has been moved into the `yaml` package namespace. +# It does not perfectly mimic its old counterpart, but should get +# close enough for anyone who's relying on it even when they shouldn't. +import yaml + +# in some circumstances, the yaml module we imoprted may be from a different version, so we need +# to tread carefully when poking at it here (it may not have the attributes we expect) +if not getattr(yaml, '__with_libyaml__', False): + from sys import version_info + + exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError + raise exc("No module named '_yaml'") +else: + from yaml._yaml import * + import warnings + warnings.warn( + 'The _yaml extension module is now located at yaml._yaml' + ' and its location is subject to change. To use the' + ' LibYAML-based parser and emitter, import from `yaml`:' + ' `from yaml import CLoader as Loader, CDumper as Dumper`.', + DeprecationWarning + ) + del warnings + # Don't `del yaml` here because yaml is actually an existing + # namespace member of _yaml. + +__name__ = '_yaml' +# If the module is top-level (i.e. not a part of any specific package) +# then the attribute should be set to ''. +# https://docs.python.org/3.8/library/types.html +__package__ = '' diff --git a/kbidu_env/Lib/site-packages/_yaml/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/_yaml/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76ac8339740e736fced7faa33a0b893c492cad97 Binary files /dev/null and b/kbidu_env/Lib/site-packages/_yaml/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/INSTALLER b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/METADATA b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..a40a10d82aa2bc2b290f56a281c6be04dddf07be --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/METADATA @@ -0,0 +1,391 @@ +Metadata-Version: 2.4 +Name: accelerate +Version: 1.14.0 +Summary: Accelerate +Home-page: https://github.com/huggingface/accelerate +Author: The Hugging Face team +Author-email: transformers@huggingface.co +License: Apache +Keywords: deep learning +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Requires-Python: >=3.10.0 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: numpy>=1.17 +Requires-Dist: packaging>=20.0 +Requires-Dist: psutil +Requires-Dist: pyyaml +Requires-Dist: torch>=2.0.0 +Requires-Dist: huggingface_hub>=0.21.0 +Requires-Dist: safetensors>=0.4.3 +Provides-Extra: quality +Requires-Dist: ruff==0.13.1; extra == "quality" +Provides-Extra: docs +Provides-Extra: test-prod +Requires-Dist: pytest>=7.2.0; extra == "test-prod" +Requires-Dist: pytest-xdist; extra == "test-prod" +Requires-Dist: pytest-subtests; extra == "test-prod" +Requires-Dist: parameterized; extra == "test-prod" +Requires-Dist: pytest-order; extra == "test-prod" +Provides-Extra: test-dev +Requires-Dist: datasets; extra == "test-dev" +Requires-Dist: diffusers; extra == "test-dev" +Requires-Dist: evaluate; extra == "test-dev" +Requires-Dist: torchdata>=0.8.0; extra == "test-dev" +Requires-Dist: torchpippy>=0.2.0; extra == "test-dev" +Requires-Dist: transformers; extra == "test-dev" +Requires-Dist: scipy; extra == "test-dev" +Requires-Dist: scikit-learn; extra == "test-dev" +Requires-Dist: tqdm; extra == "test-dev" +Requires-Dist: bitsandbytes; extra == "test-dev" +Requires-Dist: timm; extra == "test-dev" +Provides-Extra: testing +Requires-Dist: pytest>=7.2.0; extra == "testing" +Requires-Dist: pytest-xdist; extra == "testing" +Requires-Dist: pytest-subtests; extra == "testing" +Requires-Dist: parameterized; extra == "testing" +Requires-Dist: pytest-order; extra == "testing" +Requires-Dist: datasets; extra == "testing" +Requires-Dist: diffusers; extra == "testing" +Requires-Dist: evaluate; extra == "testing" +Requires-Dist: torchdata>=0.8.0; extra == "testing" +Requires-Dist: torchpippy>=0.2.0; extra == "testing" +Requires-Dist: transformers; extra == "testing" +Requires-Dist: scipy; extra == "testing" +Requires-Dist: scikit-learn; extra == "testing" +Requires-Dist: tqdm; extra == "testing" +Requires-Dist: bitsandbytes; extra == "testing" +Requires-Dist: timm; extra == "testing" +Provides-Extra: deepspeed +Requires-Dist: deepspeed; extra == "deepspeed" +Provides-Extra: rich +Requires-Dist: rich; extra == "rich" +Provides-Extra: test-fp8 +Requires-Dist: torchao; extra == "test-fp8" +Provides-Extra: test-trackers +Requires-Dist: wandb; extra == "test-trackers" +Requires-Dist: tensorboard; extra == "test-trackers" +Requires-Dist: dvclive; extra == "test-trackers" +Requires-Dist: matplotlib; extra == "test-trackers" +Requires-Dist: swanlab[dashboard]; extra == "test-trackers" +Requires-Dist: trackio; extra == "test-trackers" +Provides-Extra: dev +Requires-Dist: ruff==0.13.1; extra == "dev" +Requires-Dist: pytest>=7.2.0; extra == "dev" +Requires-Dist: pytest-xdist; extra == "dev" +Requires-Dist: pytest-subtests; extra == "dev" +Requires-Dist: parameterized; extra == "dev" +Requires-Dist: pytest-order; extra == "dev" +Requires-Dist: datasets; extra == "dev" +Requires-Dist: diffusers; extra == "dev" +Requires-Dist: evaluate; extra == "dev" +Requires-Dist: torchdata>=0.8.0; extra == "dev" +Requires-Dist: torchpippy>=0.2.0; extra == "dev" +Requires-Dist: transformers; extra == "dev" +Requires-Dist: scipy; extra == "dev" +Requires-Dist: scikit-learn; extra == "dev" +Requires-Dist: tqdm; extra == "dev" +Requires-Dist: bitsandbytes; extra == "dev" +Requires-Dist: timm; extra == "dev" +Requires-Dist: rich; extra == "dev" +Provides-Extra: sagemaker +Requires-Dist: sagemaker; extra == "sagemaker" +Dynamic: author +Dynamic: author-email +Dynamic: classifier +Dynamic: description +Dynamic: description-content-type +Dynamic: home-page +Dynamic: keywords +Dynamic: license +Dynamic: license-file +Dynamic: provides-extra +Dynamic: requires-dist +Dynamic: requires-python +Dynamic: summary + + + +

+
+ +
+

+ +

+ + License + Documentation + GitHub release + Contributor Covenant +

+ +

+

Run your *raw* PyTorch training script on any kind of device +

+ +

+ +

+ +## Easy to integrate + +🤗 Accelerate was created for PyTorch users who like to write the training loop of PyTorch models but are reluctant to write and maintain the boilerplate code needed to use multi-GPUs/TPU/fp16. + +🤗 Accelerate abstracts exactly and only the boilerplate code related to multi-GPUs/TPU/fp16 and leaves the rest of your code unchanged. + +Here is an example: + +```diff + import torch + import torch.nn.functional as F + from datasets import load_dataset ++ from accelerate import Accelerator + ++ accelerator = Accelerator() +- device = 'cpu' ++ device = accelerator.device + + model = torch.nn.Transformer().to(device) + optimizer = torch.optim.Adam(model.parameters()) + + dataset = load_dataset('my_dataset') + data = torch.utils.data.DataLoader(dataset, shuffle=True) + ++ model, optimizer, data = accelerator.prepare(model, optimizer, data) + + model.train() + for epoch in range(10): + for source, targets in data: + source = source.to(device) + targets = targets.to(device) + + optimizer.zero_grad() + + output = model(source) + loss = F.cross_entropy(output, targets) + +- loss.backward() ++ accelerator.backward(loss) + + optimizer.step() +``` + +As you can see in this example, by adding 5-lines to any standard PyTorch training script you can now run on any kind of single or distributed node setting (single CPU, single GPU, multi-GPUs and TPUs) as well as with or without mixed precision (fp8, fp16, bf16). + +In particular, the same code can then be run without modification on your local machine for debugging or your training environment. + +🤗 Accelerate even handles the device placement for you (which requires a few more changes to your code, but is safer in general), so you can even simplify your training loop further: + +```diff + import torch + import torch.nn.functional as F + from datasets import load_dataset ++ from accelerate import Accelerator + +- device = 'cpu' ++ accelerator = Accelerator() + +- model = torch.nn.Transformer().to(device) ++ model = torch.nn.Transformer() + optimizer = torch.optim.Adam(model.parameters()) + + dataset = load_dataset('my_dataset') + data = torch.utils.data.DataLoader(dataset, shuffle=True) + ++ model, optimizer, data = accelerator.prepare(model, optimizer, data) + + model.train() + for epoch in range(10): + for source, targets in data: +- source = source.to(device) +- targets = targets.to(device) + + optimizer.zero_grad() + + output = model(source) + loss = F.cross_entropy(output, targets) + +- loss.backward() ++ accelerator.backward(loss) + + optimizer.step() +``` + +Want to learn more? Check out the [documentation](https://huggingface.co/docs/accelerate) or have a look at our [examples](https://github.com/huggingface/accelerate/tree/main/examples). + +## Launching script + +🤗 Accelerate also provides an optional CLI tool that allows you to quickly configure and test your training environment before launching the scripts. No need to remember how to use `torch.distributed.run` or to write a specific launcher for TPU training! +On your machine(s) just run: + +```bash +accelerate config +``` + +and answer the questions asked. This will generate a config file that will be used automatically to properly set the default options when doing + +```bash +accelerate launch my_script.py --args_to_my_script +``` + +For instance, here is how you would run the GLUE example on the MRPC task (from the root of the repo): + +```bash +accelerate launch examples/nlp_example.py +``` + +This CLI tool is **optional**, and you can still use `python my_script.py` or `python -m torchrun my_script.py` at your convenience. + +You can also directly pass in the arguments you would to `torchrun` as arguments to `accelerate launch` if you wish to not run` accelerate config`. + +For example, here is how to launch on two GPUs: + +```bash +accelerate launch --multi_gpu --num_processes 2 examples/nlp_example.py +``` + +To learn more, check the CLI documentation available [here](https://huggingface.co/docs/accelerate/package_reference/cli). + +Or view the configuration zoo [here](https://github.com/huggingface/accelerate/blob/main/examples/config_yaml_templates/) + +## Launching multi-CPU run using MPI + +🤗 Here is another way to launch multi-CPU run using MPI. You can learn how to install Open MPI on [this page](https://www.open-mpi.org/faq/?category=building#easy-build). You can use Intel MPI or MVAPICH as well. +Once you have MPI setup on your cluster, just run: +```bash +accelerate config +``` +Answer the questions that are asked, selecting to run using multi-CPU, and answer "yes" when asked if you want accelerate to launch mpirun. +Then, use `accelerate launch` with your script like: +```bash +accelerate launch examples/nlp_example.py +``` +Alternatively, you can use mpirun directly, without using the CLI like: +```bash +mpirun -np 2 python examples/nlp_example.py +``` + +## Launching training using DeepSpeed + +🤗 Accelerate supports training on single/multiple GPUs using DeepSpeed. To use it, you don't need to change anything in your training code; you can set everything using just `accelerate config`. However, if you desire to tweak your DeepSpeed related args from your Python script, we provide you the `DeepSpeedPlugin`. + +```python +from accelerate import Accelerator, DeepSpeedPlugin + +# deepspeed needs to know your gradient accumulation steps beforehand, so don't forget to pass it +# Remember you still need to do gradient accumulation by yourself, just like you would have done without deepspeed +deepspeed_plugin = DeepSpeedPlugin(zero_stage=2, gradient_accumulation_steps=2) +accelerator = Accelerator(mixed_precision='fp16', deepspeed_plugin=deepspeed_plugin) + +# How to save your 🤗 Transformer? +accelerator.wait_for_everyone() +unwrapped_model = accelerator.unwrap_model(model) +unwrapped_model.save_pretrained(save_dir, save_function=accelerator.save, state_dict=accelerator.get_state_dict(model)) +``` + +Note: DeepSpeed support is experimental for now. In case you get into some problem, please open an issue. + +## Launching your training from a notebook + +🤗 Accelerate also provides a `notebook_launcher` function you can use in a notebook to launch a distributed training. This is especially useful for Colab or Kaggle notebooks with a TPU backend. Just define your training loop in a `training_function` then in your last cell, add: + +```python +from accelerate import notebook_launcher + +notebook_launcher(training_function) +``` + +An example can be found in [this notebook](https://github.com/huggingface/notebooks/blob/main/examples/accelerate_examples/simple_nlp_example.ipynb). [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/accelerate_examples/simple_nlp_example.ipynb) + +## Why should I use 🤗 Accelerate? + +You should use 🤗 Accelerate when you want to easily run your training scripts in a distributed environment without having to renounce full control over your training loop. This is not a high-level framework above PyTorch, just a thin wrapper so you don't have to learn a new library. In fact, the whole API of 🤗 Accelerate is in one class, the `Accelerator` object. + +## Why shouldn't I use 🤗 Accelerate? + +You shouldn't use 🤗 Accelerate if you don't want to write a training loop yourself. There are plenty of high-level libraries above PyTorch that will offer you that, 🤗 Accelerate is not one of them. + +## Frameworks using 🤗 Accelerate + +If you like the simplicity of 🤗 Accelerate but would prefer a higher-level abstraction around its capabilities, some frameworks and libraries that are built on top of 🤗 Accelerate are listed below: + +* [Amphion](https://github.com/open-mmlab/Amphion) is a toolkit for Audio, Music, and Speech Generation. Its purpose is to support reproducible research and help junior researchers and engineers get started in the field of audio, music, and speech generation research and development. +* [Animus](https://github.com/Scitator/animus) is a minimalistic framework to run machine learning experiments. Animus highlights common "breakpoints" in ML experiments and provides a unified interface for them within [IExperiment](https://github.com/Scitator/animus/blob/main/animus/core.py#L76). +* [Catalyst](https://github.com/catalyst-team/catalyst#getting-started) is a PyTorch framework for Deep Learning Research and Development. It focuses on reproducibility, rapid experimentation, and codebase reuse so you can create something new rather than write yet another train loop. Catalyst provides a [Runner](https://catalyst-team.github.io/catalyst/api/core.html#runner) to connect all parts of the experiment: hardware backend, data transformations, model training, and inference logic. +* [fastai](https://github.com/fastai/fastai#installing) is a PyTorch framework for Deep Learning that simplifies training fast and accurate neural nets using modern best practices. fastai provides a [Learner](https://docs.fast.ai/learner.html#Learner) to handle the training, fine-tuning, and inference of deep learning algorithms. +* [Finetuner](https://github.com/jina-ai/finetuner) is a service that enables models to create higher-quality embeddings for semantic search, visual similarity search, cross-modal text<->image search, recommendation systems, clustering, duplication detection, anomaly detection, or other uses. +* [InvokeAI](https://github.com/invoke-ai/InvokeAI) is a creative engine for Stable Diffusion models, offering industry-leading WebUI, terminal usage support, and serves as the foundation for many commercial products. +* [Kornia](https://kornia.readthedocs.io/en/latest/get-started/introduction.html) is a differentiable library that allows classical computer vision to be integrated into deep learning models. Kornia provides a [Trainer](https://kornia.readthedocs.io/en/latest/x.html#kornia.x.Trainer) with the specific purpose to train and fine-tune the supported deep learning algorithms within the library. +* [Open Assistant](https://projects.laion.ai/Open-Assistant/) is a chat-based assistant that understands tasks, can interact with their party systems, and retrieve information dynamically to do so. +* [pytorch-accelerated](https://github.com/Chris-hughes10/pytorch-accelerated) is a lightweight training library, with a streamlined feature set centered around a general-purpose [Trainer](https://pytorch-accelerated.readthedocs.io/en/latest/trainer.html), that places a huge emphasis on simplicity and transparency; enabling users to understand exactly what is going on under the hood, but without having to write and maintain the boilerplate themselves! +* [Stable Diffusion web UI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) is an open-source browser-based easy-to-use interface based on the Gradio library for Stable Diffusion. +* [torchkeras](https://github.com/lyhue1991/torchkeras) is a simple tool for training pytorch model just in a keras style, a dynamic and beautiful plot is provided in notebook to monitor your loss or metric. +* [transformers](https://github.com/huggingface/transformers) as a tool for helping train state-of-the-art machine learning models in PyTorch, Tensorflow, and JAX. (Accelerate is the backend for the PyTorch side). + + +## Installation + +This repository is tested on Python 3.8+ and PyTorch 1.10.0+ + +You should install 🤗 Accelerate in a [virtual environment](https://docs.python.org/3/library/venv.html). If you're unfamiliar with Python virtual environments, check out the [user guide](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/). + +First, create a virtual environment with the version of Python you're going to use and activate it. + +Then, you will need to install PyTorch: refer to the [official installation page](https://pytorch.org/get-started/locally/#start-locally) regarding the specific install command for your platform. Then 🤗 Accelerate can be installed using pip as follows: + +```bash +pip install accelerate +``` + +## Supported integrations + +- CPU only +- multi-CPU on one node (machine) +- multi-CPU on several nodes (machines) +- single GPU +- multi-GPU on one node (machine) +- multi-GPU on several nodes (machines) +- TPU +- FP16/BFloat16 mixed precision +- FP8 mixed precision with [Transformer Engine](https://github.com/NVIDIA/TransformerEngine) or [MS-AMP](https://github.com/Azure/MS-AMP/) +- DeepSpeed support (Experimental) +- PyTorch Fully Sharded Data Parallel (FSDP) support (Experimental) +- Megatron-LM support (Experimental) + +## Citing 🤗 Accelerate + +If you use 🤗 Accelerate in your publication, please cite it by using the following BibTeX entry. + +```bibtex +@Misc{accelerate, + title = {Accelerate: Training and inference at scale made simple, efficient and adaptable.}, + author = {Sylvain Gugger and Lysandre Debut and Thomas Wolf and Philipp Schmid and Zachary Mueller and Sourab Mangrulkar and Marc Sun and Benjamin Bossan}, + howpublished = {\url{https://github.com/huggingface/accelerate}}, + year = {2022} +} +``` diff --git a/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/RECORD b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..60b1528acac40c5618c09fc9d765f8c2c3bcd470 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/RECORD @@ -0,0 +1,181 @@ +../../Scripts/accelerate-config.exe,sha256=wKvS_DkZK78IywEXo_TSgm5DOxcgETs7hkQmsbAE-LM,108383 +../../Scripts/accelerate-estimate-memory.exe,sha256=NN2bPuvcYHYTujgAjcQ9IIiS0QkNeaJ2L-CU1b3qeus,108385 +../../Scripts/accelerate-launch.exe,sha256=C_S1Su5cZ9fE1B21O9c3CM5MakXPhk5-TriJeiP29PY,108383 +../../Scripts/accelerate-merge-weights.exe,sha256=ZlU1KlslRHBdOM7zfyfbEjsq7L_8hIVsMVpBIXwYahk,108382 +../../Scripts/accelerate.exe,sha256=IOAd6K4WM9VkoYK_Vcepfa-tTdYmMt4C2P_KhFXY1UA,108391 +accelerate-1.14.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +accelerate-1.14.0.dist-info/METADATA,sha256=7RwN2cyi0USbuufWQSCEMfX8zH1eBuIL7IAsHYE0D2w,19552 +accelerate-1.14.0.dist-info/RECORD,, +accelerate-1.14.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +accelerate-1.14.0.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91 +accelerate-1.14.0.dist-info/entry_points.txt,sha256=Vpy8gUGfZ-1VnM2229fb8CpJNLBdMH_wtJ9PQ7b_2tQ,296 +accelerate-1.14.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357 +accelerate-1.14.0.dist-info/top_level.txt,sha256=esVfdxTidsjQ90zsN_rPpjLFJ4ijRlx4mnLrG09hlt4,11 +accelerate/__init__.py,sha256=eeq-6vAFuLD5jcgzYQ6MBz8z6WU_NjDE7SLDCAgFLSc,1555 +accelerate/__pycache__/__init__.cpython-314.pyc,, +accelerate/__pycache__/accelerator.cpython-314.pyc,, +accelerate/__pycache__/big_modeling.cpython-314.pyc,, +accelerate/__pycache__/checkpointing.cpython-314.pyc,, +accelerate/__pycache__/data_loader.cpython-314.pyc,, +accelerate/__pycache__/hooks.cpython-314.pyc,, +accelerate/__pycache__/inference.cpython-314.pyc,, +accelerate/__pycache__/launchers.cpython-314.pyc,, +accelerate/__pycache__/local_sgd.cpython-314.pyc,, +accelerate/__pycache__/logging.cpython-314.pyc,, +accelerate/__pycache__/memory_utils.cpython-314.pyc,, +accelerate/__pycache__/optimizer.cpython-314.pyc,, +accelerate/__pycache__/parallelism_config.cpython-314.pyc,, +accelerate/__pycache__/scheduler.cpython-314.pyc,, +accelerate/__pycache__/state.cpython-314.pyc,, +accelerate/__pycache__/tracking.cpython-314.pyc,, +accelerate/accelerator.py,sha256=RwiOCrO_Ie7JfhavoUWV4dtRH26tmrhcTqpfb2b-XmE,197583 +accelerate/big_modeling.py,sha256=TawUU0RaWZipSQMcys2PKpFWGnFSoA_F4YPERa446x8,37573 +accelerate/checkpointing.py,sha256=HpNLk1v6MI6QLb-jiWG0xMQCHcYscEfZ_kNJRGnnO-Y,14472 +accelerate/commands/__init__.py,sha256=m1PPTDT4ziIAvM0-FDSgIMIZ69Konn126s6LwuzH6v8,606 +accelerate/commands/__pycache__/__init__.cpython-314.pyc,, +accelerate/commands/__pycache__/accelerate_cli.cpython-314.pyc,, +accelerate/commands/__pycache__/env.cpython-314.pyc,, +accelerate/commands/__pycache__/estimate.cpython-314.pyc,, +accelerate/commands/__pycache__/launch.cpython-314.pyc,, +accelerate/commands/__pycache__/merge.cpython-314.pyc,, +accelerate/commands/__pycache__/test.cpython-314.pyc,, +accelerate/commands/__pycache__/to_fsdp2.cpython-314.pyc,, +accelerate/commands/__pycache__/tpu.cpython-314.pyc,, +accelerate/commands/__pycache__/utils.cpython-314.pyc,, +accelerate/commands/accelerate_cli.py,sha256=SkwFad6Z1ZsGjtm7TiXFq8je-akshp_0WxX_6rGSBw8,1972 +accelerate/commands/config/__init__.py,sha256=iJK8dgj3pc5Vdr1E7UuGoFu-BlybyXLxYDoTg9gXngE,1645 +accelerate/commands/config/__pycache__/__init__.cpython-314.pyc,, +accelerate/commands/config/__pycache__/cluster.cpython-314.pyc,, +accelerate/commands/config/__pycache__/config.cpython-314.pyc,, +accelerate/commands/config/__pycache__/config_args.cpython-314.pyc,, +accelerate/commands/config/__pycache__/config_utils.cpython-314.pyc,, +accelerate/commands/config/__pycache__/default.cpython-314.pyc,, +accelerate/commands/config/__pycache__/sagemaker.cpython-314.pyc,, +accelerate/commands/config/__pycache__/update.cpython-314.pyc,, +accelerate/commands/config/cluster.py,sha256=2IyA3-Jd9jH9PD5dsv8uD99h1Mo0vBJ8A_SqyY-BqQ0,40572 +accelerate/commands/config/config.py,sha256=FuRlQvOjgATEtyqOSsGD-KEtOCvACOHjs2C-krrtldk,3035 +accelerate/commands/config/config_args.py,sha256=KsnR-JTtZHgEQ3AFD7mvqrP7TPYf9dnc45ichcN3_OI,10145 +accelerate/commands/config/config_utils.py,sha256=5jnBxHxzaKKfSqBOeznJk4Js4bHE-6u1kq2KwuunK58,3321 +accelerate/commands/config/default.py,sha256=c0xKShclwHWEIuwMe5uNkb2AplUyHu20pkHlPUemGsw,6559 +accelerate/commands/config/sagemaker.py,sha256=SRLup9XrV_Z-23A3d8gZbkqbonDcscMDDmbpmytCzbY,10636 +accelerate/commands/config/update.py,sha256=0Et2qKKgoeisVQixsJKSqp4dLzSofbZ2jXmAPgJP-DQ,2396 +accelerate/commands/env.py,sha256=Ny5V2z8TFHKvSZvuEaaJf233N3SDkQfjPv0FJ7U8TAw,4586 +accelerate/commands/estimate.py,sha256=TU98i7qU76kYZBA1PBjy38RXatYkIY_e_oncOpWozeo,12667 +accelerate/commands/launch.py,sha256=WTkLWqPGubEREorn0bwKXVc59bv1bE_Kw7tRXRifE9g,55380 +accelerate/commands/menu/__init__.py,sha256=uqSlBM0TFHBwzdv3p3SXfpAk1lZFp4h1a7mbBdscPHs,645 +accelerate/commands/menu/__pycache__/__init__.cpython-314.pyc,, +accelerate/commands/menu/__pycache__/cursor.cpython-314.pyc,, +accelerate/commands/menu/__pycache__/helpers.cpython-314.pyc,, +accelerate/commands/menu/__pycache__/input.cpython-314.pyc,, +accelerate/commands/menu/__pycache__/keymap.cpython-314.pyc,, +accelerate/commands/menu/__pycache__/selection_menu.cpython-314.pyc,, +accelerate/commands/menu/cursor.py,sha256=-lmpJVAzvNc0c3EOtSuLoKB59zqylVCbYyWLPnrOmvQ,2028 +accelerate/commands/menu/helpers.py,sha256=KrSB5fJjH4MUEUAQJ6bYaN16AYcnl9UalDrPD3DYeeg,1483 +accelerate/commands/menu/input.py,sha256=T8Mdd-Y_OURgqfDV9qZh4Wf6hmT22AneNtJzj4JA1Rk,2512 +accelerate/commands/menu/keymap.py,sha256=eXj-suyYs1m5dEHoUKN4mKAMLc8DWHnwhP6G6JSU0jQ,4086 +accelerate/commands/menu/selection_menu.py,sha256=iylPHUmEemEhn6gwidbgdTGOrCJiR3Il1EjqiYbnues,4959 +accelerate/commands/merge.py,sha256=3Uo14LADvk8RJwZ_i6kIsGwWB3Ul5y8Uq47oHkDYqKU,2387 +accelerate/commands/test.py,sha256=YrPYEaAACOGZ6btn2MV6NbMSEdBUcMWADLbQWaZSHtk,2149 +accelerate/commands/to_fsdp2.py,sha256=xT0RbJx5nFiYK9MW-vaCVeO7Zs33fsqZsHm2zFoXCu8,6140 +accelerate/commands/tpu.py,sha256=KyxDP7IuveidZrbW4rx2s8Ku3o_ptI6tzwr_R7ck0os,5548 +accelerate/commands/utils.py,sha256=aT8xUCe2pCkFII7yZxcfaohEjgBAzMUM7WiD4UuWSOY,4150 +accelerate/data_loader.py,sha256=4IxfXNP3qLiq7OA61R8cN5lOOQ-vR9NoKBrUnDG8hNE,66830 +accelerate/hooks.py,sha256=hneJpxQ3bOHRyPOp-Iu-58-cRBTJrKu7RDuvLYiRkoI,35083 +accelerate/inference.py,sha256=8M1LRCfXSstOUhS8O6UlCipQ14cTQgvAqWqDXQEQn50,7684 +accelerate/launchers.py,sha256=yike9Ef2lMRNya98x9xwEkcQpnMlPyPv68woF25efIg,15147 +accelerate/local_sgd.py,sha256=PS1P8TnsRJKdYDwnOw2HoQZI8gEC8ntJuI7l79j07TY,4218 +accelerate/logging.py,sha256=lkS2iA1mPJeKvbAOdf5FwIYP0TzWTjZFwMOmmDiTrQ8,5233 +accelerate/memory_utils.py,sha256=3R5LoeHl6GgTZ-IMPrDZMdaEehWarGdPqODushb-6pg,862 +accelerate/optimizer.py,sha256=p5qfCOMi8DAjvx6eECypo4ej3pvmQ_U_bgmrEAKc4hY,8158 +accelerate/parallelism_config.py,sha256=YzuU70g-xz1iKN7xMt-DsAOe1jqnTY6mIRQG0xuwdeE,17071 +accelerate/scheduler.py,sha256=des_4M_Tt1W8gCYZZbLla0GHBEgJY3Wx2EGBQPTzeiY,4238 +accelerate/state.py,sha256=GOCiw4v8dFp73rWHirIyxVtKglbYVmmaKCUFFonn3OI,58188 +accelerate/test_utils/__init__.py,sha256=E347JKLya-_vi4IbWhdsAVQ0FCudtcaD7ErX2bh6RC0,1826 +accelerate/test_utils/__pycache__/__init__.cpython-314.pyc,, +accelerate/test_utils/__pycache__/examples.cpython-314.pyc,, +accelerate/test_utils/__pycache__/testing.cpython-314.pyc,, +accelerate/test_utils/__pycache__/training.cpython-314.pyc,, +accelerate/test_utils/examples.py,sha256=kSXJrx6OhUghHGzqLnCwqdsDUufMjmB2DPQyQSfZHvQ,7301 +accelerate/test_utils/scripts/__init__.py,sha256=m1PPTDT4ziIAvM0-FDSgIMIZ69Konn126s6LwuzH6v8,606 +accelerate/test_utils/scripts/__pycache__/__init__.cpython-314.pyc,, +accelerate/test_utils/scripts/__pycache__/test_cli.cpython-314.pyc,, +accelerate/test_utils/scripts/__pycache__/test_ddp_comm_hook.cpython-314.pyc,, +accelerate/test_utils/scripts/__pycache__/test_distributed_data_loop.cpython-314.pyc,, +accelerate/test_utils/scripts/__pycache__/test_merge_weights.cpython-314.pyc,, +accelerate/test_utils/scripts/__pycache__/test_notebook.cpython-314.pyc,, +accelerate/test_utils/scripts/__pycache__/test_ops.cpython-314.pyc,, +accelerate/test_utils/scripts/__pycache__/test_script.cpython-314.pyc,, +accelerate/test_utils/scripts/__pycache__/test_sync.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/__init__.py,sha256=m1PPTDT4ziIAvM0-FDSgIMIZ69Konn126s6LwuzH6v8,606 +accelerate/test_utils/scripts/external_deps/__pycache__/__init__.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/__pycache__/test_checkpointing.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/__pycache__/test_ds_alst_ulysses_sp.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/__pycache__/test_ds_multiple_model.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/__pycache__/test_metrics.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/__pycache__/test_peak_memory_usage.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/__pycache__/test_performance.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/__pycache__/test_pippy.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/__pycache__/test_zero3_integration.cpython-314.pyc,, +accelerate/test_utils/scripts/external_deps/test_checkpointing.py,sha256=Y_ZaRE9o1c8dN7Xj3VMLtIemHuydVl3sRPPgpwnN_hk,10707 +accelerate/test_utils/scripts/external_deps/test_ds_alst_ulysses_sp.py,sha256=JQE9g4b2wB-_KMdUcR_RzLUJ5iLMZiowR1tu7bbAImo,4077 +accelerate/test_utils/scripts/external_deps/test_ds_multiple_model.py,sha256=e4wTB0ikmBQ81BGTLU0gop1UFMvPpt1GiJw4NX90TkM,13773 +accelerate/test_utils/scripts/external_deps/test_metrics.py,sha256=gz6MnxF8U1z-Br2o1uhOZbP8HbNPWGURg-p9ZuTcdpQ,12176 +accelerate/test_utils/scripts/external_deps/test_peak_memory_usage.py,sha256=1ikv3mBgIEuvn0GUmSIIPFYNlmrNxsy9Ept4nNj_iTk,12933 +accelerate/test_utils/scripts/external_deps/test_performance.py,sha256=MqOmhmdWxvYeB6xB1Vfg83XDgGxpnHfnWj-mTBoAoT4,12101 +accelerate/test_utils/scripts/external_deps/test_pippy.py,sha256=rYF5bqSCe_pfylxDv2_2Q7S2mHIb9_6fGns-IcLgE_E,4789 +accelerate/test_utils/scripts/external_deps/test_zero3_integration.py,sha256=-fc3l3PSAplU82FLaeIWCAJdAVBsT72vBQpdJfZ2Hsw,1666 +accelerate/test_utils/scripts/test_cli.py,sha256=-p78NTJgKdeTNDqyrQuGEZZqMpnzdUHPyWZ05FMl5EI,1081 +accelerate/test_utils/scripts/test_ddp_comm_hook.py,sha256=k_-2MBjLKNdMGIcneTbuGd84K05Wp1GEQX6DUVF9UBw,3566 +accelerate/test_utils/scripts/test_distributed_data_loop.py,sha256=OPs59U7Zp0OB3CO6X0orgvTv2gCbQBzHiFsCsTB3ME0,16020 +accelerate/test_utils/scripts/test_merge_weights.py,sha256=r1oqPOMOqLe5zWenWWTyl9cDbbkFJadpNNiPNBlgR18,5982 +accelerate/test_utils/scripts/test_notebook.py,sha256=uaIyiuv78q2eFjklUq6-_5-t4GqQXEsFe_ZVeB_MZoc,5053 +accelerate/test_utils/scripts/test_ops.py,sha256=_w0u1zTMGoB5XqDw1DBwpnEGUtRsHghg5j1OtiKK5po,6683 +accelerate/test_utils/scripts/test_script.py,sha256=aYsLQlCm0ax_rLOBqeVacauaCxefjgusPOONiiwnXJI,34740 +accelerate/test_utils/scripts/test_sync.py,sha256=Bf0JIdqdlOgDkaqtVw4m-QprTLw9IY8sKxXp0JCJSc4,19027 +accelerate/test_utils/testing.py,sha256=EnO0qXTNZYx9HrebVC9tIIdyieVGk-xwSwDCSMENtnU,30757 +accelerate/test_utils/training.py,sha256=7X6JvNeOJhkuw48Bzugexsq85a9KWBTFMZSJeizNhuc,6043 +accelerate/tracking.py,sha256=C4hHQkfXjwE7wuNCGtM2ll0Ri9NqXbYdXqqiW_kxmZo,51264 +accelerate/utils/__init__.py,sha256=HbmUqegtINt4zjR2mrPbeHjvQkni9ZV5S7gec8zU49k,8286 +accelerate/utils/__pycache__/__init__.cpython-314.pyc,, +accelerate/utils/__pycache__/ao.cpython-314.pyc,, +accelerate/utils/__pycache__/bnb.cpython-314.pyc,, +accelerate/utils/__pycache__/constants.cpython-314.pyc,, +accelerate/utils/__pycache__/dataclasses.cpython-314.pyc,, +accelerate/utils/__pycache__/deepspeed.cpython-314.pyc,, +accelerate/utils/__pycache__/environment.cpython-314.pyc,, +accelerate/utils/__pycache__/fsdp_utils.cpython-314.pyc,, +accelerate/utils/__pycache__/imports.cpython-314.pyc,, +accelerate/utils/__pycache__/launch.cpython-314.pyc,, +accelerate/utils/__pycache__/megatron_lm.cpython-314.pyc,, +accelerate/utils/__pycache__/memory.cpython-314.pyc,, +accelerate/utils/__pycache__/modeling.cpython-314.pyc,, +accelerate/utils/__pycache__/offload.cpython-314.pyc,, +accelerate/utils/__pycache__/operations.cpython-314.pyc,, +accelerate/utils/__pycache__/other.cpython-314.pyc,, +accelerate/utils/__pycache__/random.cpython-314.pyc,, +accelerate/utils/__pycache__/rich.cpython-314.pyc,, +accelerate/utils/__pycache__/torch_xla.cpython-314.pyc,, +accelerate/utils/__pycache__/tqdm.cpython-314.pyc,, +accelerate/utils/__pycache__/transformer_engine.cpython-314.pyc,, +accelerate/utils/__pycache__/versions.cpython-314.pyc,, +accelerate/utils/ao.py,sha256=gnsP4LMmkfGUIDwxb4vJBOu9H4zBDkXgMo9MIAp_vUg,4860 +accelerate/utils/bnb.py,sha256=5O69ySpDVKXf03ujS8qoZvZAyFo05vC-ImzJwcf6Yhc,21228 +accelerate/utils/constants.py,sha256=2LUbK5PFg_TM_Yqae0Mfu-AQKsfgUJsg7FUlGRRwmrA,3662 +accelerate/utils/dataclasses.py,sha256=51J3b_wvR6FAh9B4vLZetotV3cVcjNt1gieuHkA2kPI,156000 +accelerate/utils/deepspeed.py,sha256=gt-jwOpOtRs6N4sohuSO2I3x1qLoO6uYYjnPrKp6Zk4,14796 +accelerate/utils/environment.py,sha256=WhYd0rprJ6JtDxo2v5TUilK-8m7tTgw1VEV3bIIezIQ,18579 +accelerate/utils/fsdp_utils.py,sha256=AZQGkZaOJBZDqyVxP8H8OELZ8mONLHBXQSpCFYU2Ekc,42977 +accelerate/utils/imports.py,sha256=ShSvvEWZQ5f0KTJ2DdzYmvVrWJ6U637tXOt_BGn8V-U,16064 +accelerate/utils/launch.py,sha256=O-vjgwOU-3cbrHQliEbI43R5SaMsoRBGhPGXjSgMpNM,38710 +accelerate/utils/megatron_lm.py,sha256=SenysvnX3VYTQC8DakDC3m1bfB1ud4TjX0A402K_RZA,50456 +accelerate/utils/memory.py,sha256=rPoLg3NahMvNDN7CuZxfX3UAmuARxuR3kL2Pv-H8pL0,6340 +accelerate/utils/modeling.py,sha256=P2A6WU2Vr89234G7SHCXCVsygraqq1ByjmkaKmsX24A,96226 +accelerate/utils/offload.py,sha256=N6ATpOZph7gGrZUjriicD7F6oIhZvmJND66jRyy1nlw,7845 +accelerate/utils/operations.py,sha256=l6LljUkoSRJpxXOcxyPoTGSH15Ri2J2GqS_rT_c7e3M,36945 +accelerate/utils/other.py,sha256=4fueMlyIsXw9_f4xiXjGBWbXs1K8QgFZjJbWc3x7Jp4,21668 +accelerate/utils/random.py,sha256=h-_wl1pId62RIt6VW9tt1YeSARKiMnk1Oz_0aHdAW4E,6665 +accelerate/utils/rich.py,sha256=8JZX_uGMQX-BufdXxJpdne7BWd1KyLHSgbiGxrDMYr8,847 +accelerate/utils/torch_xla.py,sha256=Pq1tuqN0X_pWDVza6YgjfO45uoJdoRVRForLeLQzFus,1908 +accelerate/utils/tqdm.py,sha256=k8e9JnieTEQHCCNBaiBys7hPxWlEbyRASdIma-qy_X8,1657 +accelerate/utils/transformer_engine.py,sha256=MFnzMxP1WB7Ucu56qXtnJPhI789zxoIG6dOmzQu3Iro,7537 +accelerate/utils/versions.py,sha256=UgmcbjBm--6CIx1ZamSAMjAK_B_2l48LbeaNygqej8M,2149 diff --git a/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/REQUESTED b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/WHEEL b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..370ec5f1aa3524bfd31a19fd21e907335f48a7f6 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (78.1.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/entry_points.txt b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b9bf6b798b250a47a3febdf0e32c88507fbf86d --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/entry_points.txt @@ -0,0 +1,6 @@ +[console_scripts] +accelerate = accelerate.commands.accelerate_cli:main +accelerate-config = accelerate.commands.config:main +accelerate-estimate-memory = accelerate.commands.estimate:main +accelerate-launch = accelerate.commands.launch:main +accelerate-merge-weights = accelerate.commands.merge:main diff --git a/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/licenses/LICENSE b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/licenses/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/top_level.txt b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..a9368375be0e0e13fdad0eea4b92541bd9e1f594 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate-1.14.0.dist-info/top_level.txt @@ -0,0 +1 @@ +accelerate diff --git a/kbidu_env/Lib/site-packages/accelerate/__init__.py b/kbidu_env/Lib/site-packages/accelerate/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a5776df54b0fc2ee8be9038d84703bb40e346560 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/__init__.py @@ -0,0 +1,51 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +__version__ = "1.14.0" + +from .accelerator import Accelerator +from .big_modeling import ( + cpu_offload, + cpu_offload_with_hook, + disk_offload, + dispatch_model, + init_empty_weights, + init_on_device, + load_checkpoint_and_dispatch, +) +from .data_loader import skip_first_batches +from .inference import prepare_pippy +from .launchers import debug_launcher, notebook_launcher +from .parallelism_config import ParallelismConfig +from .state import PartialState +from .utils import ( + AutocastKwargs, + DataLoaderConfiguration, + DDPCommunicationHookType, + DeepSpeedPlugin, + DistributedDataParallelKwargs, + DistributedType, + FullyShardedDataParallelPlugin, + GradScalerKwargs, + InitProcessGroupKwargs, + ProfileKwargs, + find_executable_batch_size, + infer_auto_device_map, + is_rich_available, + load_checkpoint_in_model, + synchronize_rng_states, +) + + +if is_rich_available(): + from .utils import rich diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..45615d2447c098cacc952e48fb6415bd21899210 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/accelerator.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/accelerator.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5389b3ce4f697ffeffdb13195f97f088719123b2 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/__pycache__/accelerator.cpython-314.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35efb2c729be25b8eb5bc7f73af8cc4282170ce4f4791210082ec9db1f4958ff +size 202584 diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/big_modeling.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/big_modeling.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1610a81d1737110aee80722cbaf1b772d93aad04 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/big_modeling.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/checkpointing.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/checkpointing.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07d41105b4b252e5e4dade89b9fd136444302b51 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/checkpointing.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/data_loader.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/data_loader.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..494615094d76c53a2161f0424073a96ff6756b3c Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/data_loader.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/hooks.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/hooks.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd52deeebd929a6be5b0fb6dec4632e30783e91c Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/hooks.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/inference.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/inference.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7df1ac53dc864f612eaa40a445c6592e63e7d12 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/inference.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/launchers.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/launchers.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b497da4f8b2738d59cdce3678f6d51775ee69dc Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/launchers.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/local_sgd.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/local_sgd.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..00197341d7656851b21bb95ca9e37e70fc01d719 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/local_sgd.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/logging.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/logging.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f08d153936f6e5450c22eb19e1115f7722c4b759 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/logging.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/memory_utils.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/memory_utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..285a4d0d7643b64996a975160965b9c0fbfd507e Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/memory_utils.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/optimizer.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/optimizer.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..436e3ee3ee3bf1ba032307aa90591430d83ea2dd Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/optimizer.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/parallelism_config.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/parallelism_config.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec7e32ea8064d5ca3043366f0eb09fdd1020625a Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/parallelism_config.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/scheduler.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/scheduler.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c2f5ea6908d7c996b7ee834bfd6f5e4996df9b27 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/scheduler.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/state.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/state.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d1b1a46726ecf0828b6c793aa06e32aae55ad2a Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/state.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/__pycache__/tracking.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/__pycache__/tracking.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d4a95aaec2194ea0475c3b98956a334187f60c93 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/__pycache__/tracking.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/accelerator.py b/kbidu_env/Lib/site-packages/accelerate/accelerator.py new file mode 100644 index 0000000000000000000000000000000000000000..66f5dcc16eee639b1415c68d06e7f5ab60ec5295 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/accelerator.py @@ -0,0 +1,4359 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import contextlib +import functools +import inspect +import json +import math +import os +import re +import shutil +import warnings +from collections import OrderedDict +from contextlib import contextmanager +from functools import partial +from types import MethodType +from typing import Any, Callable, Union + +import torch +import torch.utils.hooks as hooks + +from accelerate.utils.dataclasses import FP8BackendType + +from .big_modeling import _attach_context_parallel_hooks +from .checkpointing import load_accelerator_state, load_custom_state, save_accelerator_state, save_custom_state +from .data_loader import DataLoaderDispatcher, prepare_data_loader, skip_first_batches +from .logging import get_logger +from .optimizer import AcceleratedOptimizer +from .parallelism_config import ParallelismConfig +from .scheduler import AcceleratedScheduler +from .state import AcceleratorState, GradientState, PartialState +from .tracking import LOGGER_TYPE_TO_CLASS, GeneralTracker, filter_trackers +from .utils import ( + MODEL_NAME, + SAFE_WEIGHTS_INDEX_NAME, + SAFE_WEIGHTS_NAME, + SAFE_WEIGHTS_PATTERN_NAME, + WEIGHTS_INDEX_NAME, + WEIGHTS_NAME, + WEIGHTS_PATTERN_NAME, + AORecipeKwargs, + AutocastKwargs, + DataLoaderConfiguration, + DeepSpeedPlugin, + DistributedDataParallelKwargs, + DistributedType, + DynamoBackend, + FP8RecipeKwargs, + FullyShardedDataParallelPlugin, + GradientAccumulationPlugin, + GradScalerKwargs, + InitProcessGroupKwargs, + KwargsHandler, + LoggerType, + MegatronLMPlugin, + MSAMPRecipeKwargs, + PrecisionType, + ProfileKwargs, + ProjectConfiguration, + RNGType, + TERecipeKwargs, + TorchDynamoPlugin, + TorchTensorParallelPlugin, + apply_fp8_autowrap, + check_os_kernel, + clean_state_dict_for_safetensors, + compare_versions, + convert_model, + convert_model_to_fp8_ao, + convert_outputs_to_fp32, + ensure_weights_retied, + extract_model_from_parallel, + fsdp2_apply_ac, + fsdp2_canonicalize_names, + fsdp2_prepare_model, + fsdp2_switch_optimizer_parameters, + gather, + gather_object, + get_fsdp2_grad_scaler, + get_grad_scaler, + get_mixed_precision_context_manager, + get_pretty_name, + has_offloaded_params, + is_bf16_available, + is_bitsandbytes_multi_backend_available, + is_deepspeed_available, + is_lomo_available, + is_megatron_lm_available, + is_mlu_available, + is_msamp_available, + is_musa_available, + is_npu_available, + is_torch_version, + is_torch_xla_available, + is_torchao_available, + is_transformer_engine_available, + is_xpu_available, + load_fsdp_model, + load_fsdp_optimizer, + model_has_dtensor, + pad_across_processes, + parse_choice_from_env, + recursively_apply, + reduce, + release_memory, + save, + save_fsdp_model, + save_fsdp_optimizer, + wait_for_everyone, +) +from .utils.constants import ( + DTENSOR_PYTORCH_VERSION, + FSDP2_PYTORCH_VERSION, + FSDP_PYTORCH_VERSION, + PROFILE_PATTERN_NAME, + SCALER_NAME, +) +from .utils.modeling import get_state_dict_offloaded_model +from .utils.other import compile_regions, compile_regions_deepspeed, compile_regions_fsdp2, is_compiled_module + + +if is_deepspeed_available(): + from .utils import ( + DeepSpeedEngineWrapper, + DeepSpeedOptimizerWrapper, + DeepSpeedSchedulerWrapper, + DummyOptim, + DummyScheduler, + map_pytorch_optim_to_deepspeed, + ) + +if is_megatron_lm_available(): + from .utils import ( + MegatronEngine, + MegatronLMDummyDataLoader, + MegatronLMDummyScheduler, + MegatronLMOptimizerWrapper, + MegatronLMSchedulerWrapper, + megatron_lm_initialize, + megatron_lm_prepare_data_loader, + megatron_lm_prepare_model_optimizer_scheduler, + ) + +if torch.distributed.is_available(): + from torch.distributed.algorithms.join import Join + + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + import torch_xla.distributed.xla_multiprocessing as xmp + + +if is_npu_available(check_device=False): + import torch_npu # noqa: F401 + + +try: + from torch.optim.lr_scheduler import LRScheduler +except ImportError: + from torch.optim.lr_scheduler import _LRScheduler as LRScheduler + +logger = get_logger(__name__) + +# Sentinel values for defaults +_split_batches = object() +_dispatch_batches = object() +_even_batches = object() +_use_seedable_sampler = object() + + +class Accelerator: + """ + Creates an instance of an accelerator for distributed training or mixed precision training. + + Args: + device_placement (`bool`, *optional*, defaults to `True`): + Whether or not the accelerator should put objects on device (tensors yielded by the dataloader, model, + etc...). + mixed_precision (`str`, *optional*): + Whether or not to use mixed precision training. Choose from 'no','fp16','bf16' or 'fp8'. Will default to + the value in the environment variable `ACCELERATE_MIXED_PRECISION`, which will use the default value in the + accelerate config of the current system or the flag passed with the `accelerate.launch` command. 'fp8' + requires the installation of transformers-engine. + gradient_accumulation_steps (`int`, *optional*, default to 1): + The number of steps that should pass before gradients are accumulated. A number > 1 should be combined with + `Accelerator.accumulate`. If not passed, will default to the value in the environment variable + `ACCELERATE_GRADIENT_ACCUMULATION_STEPS`. Can also be configured through a `GradientAccumulationPlugin`. + cpu (`bool`, *optional*): + Whether or not to force the script to execute on CPU. Will ignore GPU available if set to `True` and force + the execution on one process only. + dataloader_config (`DataLoaderConfiguration`, *optional*): + A configuration for how the dataloaders should be handled in distributed scenarios. + deepspeed_plugin ([`~utils.DeepSpeedPlugin`] or dict of `str`: [`~utils.DeepSpeedPlugin`], *optional*): + Tweak your DeepSpeed related args using this argument. This argument is optional and can be configured + directly using *accelerate config*. If using multiple plugins, use the configured `key` property of each + plugin to access them from `accelerator.state.get_deepspeed_plugin(key)`. Alias for `deepspeed_plugins`. + fsdp_plugin ([`~utils.FullyShardedDataParallelPlugin`], *optional*): + Tweak your FSDP related args using this argument. This argument is optional and can be configured directly + using *accelerate config* + torch_tp_plugin ([`~utils.TorchTensorParallelPlugin`], *optional*): + Deprecated: use `parallelism_config` with `tp_size` instead. + megatron_lm_plugin ([`~utils.MegatronLMPlugin`], *optional*): + Tweak your MegatronLM related args using this argument. This argument is optional and can be configured + directly using *accelerate config* + rng_types (list of `str` or [`~utils.RNGType`]): + The list of random number generators to synchronize at the beginning of each iteration in your prepared + dataloaders. Should be one or several of: + + - `"torch"`: the base torch random number generator + - `"cuda"`: the CUDA random number generator (GPU only) + - `"xla"`: the XLA random number generator (TPU only) + - `"generator"`: the `torch.Generator` of the sampler (or batch sampler if there is no sampler in your + dataloader) or of the iterable dataset (if it exists) if the underlying dataset is of that type. + + Will default to `["torch"]` for PyTorch versions <=1.5.1 and `["generator"]` for PyTorch versions >= 1.6. + log_with (list of `str`, [`~utils.LoggerType`] or [`~tracking.GeneralTracker`], *optional*): + A list of loggers to be setup for experiment tracking. Should be one or several of: + + - `"all"` + - `"tensorboard"` + - `"wandb"` + - `"trackio"` + - `"aim"` + - `"comet_ml"` + - `"mlflow"` + - `"dvclive"` + - `"swanlab"` + If `"all"` is selected, will pick up all available trackers in the environment and initialize them. Can + also accept implementations of `GeneralTracker` for custom trackers, and can be combined with `"all"`. + project_config ([`~utils.ProjectConfiguration`], *optional*): + A configuration for how saving the state can be handled. + project_dir (`str`, `os.PathLike`, *optional*): + A path to a directory for storing data such as logs of locally-compatible loggers and potentially saved + checkpoints. + step_scheduler_with_optimizer (`bool`, *optional*, defaults to `True`): + Set `True` if the learning rate scheduler is stepped at the same time as the optimizer, `False` if only + done under certain circumstances (at the end of each epoch, for instance). + kwargs_handlers (list of [`~utils.KwargsHandler`], *optional*) + A list of [`~utils.KwargsHandler`] to customize how the objects related to distributed training, profiling + or mixed precision are created. See [kwargs](kwargs) for more information. + dynamo_backend (`str` or [`~utils.DynamoBackend`], *optional*, defaults to `"no"`): + Set to one of the possible dynamo backends to optimize your training with torch dynamo. + dynamo_plugin ([`~utils.TorchDynamoPlugin`], *optional*): + A configuration for how torch dynamo should be handled, if more tweaking than just the `backend` or `mode` + is needed. + gradient_accumulation_plugin ([`~utils.GradientAccumulationPlugin`], *optional*): + A configuration for how gradient accumulation should be handled, if more tweaking than just the + `gradient_accumulation_steps` is needed. + + **Available attributes:** + + - **device** (`torch.device`) -- The device to use. + - **distributed_type** ([`~utils.DistributedType`]) -- The distributed training configuration. + - **local_process_index** (`int`) -- The process index on the current machine. + - **mixed_precision** (`str`) -- The configured mixed precision mode. + - **num_processes** (`int`) -- The total number of processes used for training. + - **optimizer_step_was_skipped** (`bool`) -- Whether or not the optimizer update was skipped (because of + gradient overflow in mixed precision), in which + case the learning rate should not be changed. + - **process_index** (`int`) -- The overall index of the current process among all processes. + - **state** ([`~state.AcceleratorState`]) -- The distributed setup state. + - **sync_gradients** (`bool`) -- Whether the gradients are currently being synced across all processes. + - **use_distributed** (`bool`) -- Whether the current configuration is for distributed training. + """ + + def __init__( + self, + device_placement: bool = True, + split_batches: bool = _split_batches, + mixed_precision: PrecisionType | str | None = None, + gradient_accumulation_steps: int = 1, + cpu: bool = False, + dataloader_config: DataLoaderConfiguration | None = None, + deepspeed_plugin: DeepSpeedPlugin | dict[str, DeepSpeedPlugin] | None = None, + fsdp_plugin: FullyShardedDataParallelPlugin | None = None, + torch_tp_plugin: TorchTensorParallelPlugin | None = None, # Deprecate later, warning in `post_init` + megatron_lm_plugin: MegatronLMPlugin | None = None, + rng_types: list[str | RNGType] | None = None, + log_with: str | LoggerType | GeneralTracker | list[str | LoggerType | GeneralTracker] | None = None, + project_dir: str | os.PathLike | None = None, + project_config: ProjectConfiguration | None = None, + gradient_accumulation_plugin: GradientAccumulationPlugin | None = None, + step_scheduler_with_optimizer: bool = True, + kwargs_handlers: list[KwargsHandler] | None = None, + dynamo_backend: DynamoBackend | str | None = None, + dynamo_plugin: TorchDynamoPlugin | None = None, + deepspeed_plugins: DeepSpeedPlugin | dict[str, DeepSpeedPlugin] | None = None, + parallelism_config: ParallelismConfig | None = None, + ): + self.trackers = [] + if project_config is not None: + self.project_configuration = project_config + else: + self.project_configuration = ProjectConfiguration(project_dir=project_dir) + if project_dir is not None and self.project_dir is None: + self.project_configuration.set_directories(project_dir) + + if mixed_precision is not None: + mixed_precision = str(mixed_precision) + if mixed_precision not in PrecisionType: + raise ValueError( + f"Unknown mixed_precision mode: {mixed_precision}. Choose between {PrecisionType.list()}" + ) + if torch_tp_plugin is not None: + warnings.warn( + "`TorchTensorParallelPlugin` is deprecated and will be removed in a future version of Accelerate. " + "Please use the `ParallelismConfig` with `tp_size` instead.", + FutureWarning, + ) + + if dynamo_plugin is not None and dynamo_backend is not None: + raise ValueError("You cannot pass in both `dynamo_plugin` and `dynamo_backend`, please only pass in one.") + if dynamo_backend is not None: + dynamo_plugin = TorchDynamoPlugin(backend=dynamo_backend) + elif dynamo_plugin is None: + dynamo_plugin = TorchDynamoPlugin() + + if deepspeed_plugins is not None and deepspeed_plugin is not None: + raise ValueError("You cannot pass in both `deepspeed_plugins` and `deepspeed_plugin`.") + elif deepspeed_plugin is not None: + deepspeed_plugins = deepspeed_plugin + + if deepspeed_plugins is None: + # First check if we're creating another `Accelerator` w/o setting `deepspeed_plugin` + if ( + AcceleratorState._shared_state != {} + and AcceleratorState().distributed_type == DistributedType.DEEPSPEED + ): + deepspeed_plugins = AcceleratorState().deepspeed_plugins + else: + # init from env variables + deepspeed_plugins = ( + DeepSpeedPlugin() + if os.environ.get("ACCELERATE_USE_DEEPSPEED", "false").lower() == "true" + else None + ) + else: + # If we're creating a second `Accelerator`, users shouldn't be passing in a `deepspeed_plugin` + if ( + AcceleratorState._shared_state != {} + and AcceleratorState().distributed_type == DistributedType.DEEPSPEED + and AcceleratorState().deepspeed_plugins is not None + ): + raise NotImplementedError( + "You cannot pass in a `deepspeed_plugin` when creating a second `Accelerator`. " + "Please make sure the first `Accelerator` is initialized with all the plugins you want to use." + ) + if isinstance(deepspeed_plugins, dict): + for plugin in deepspeed_plugins.values(): + if not isinstance(plugin, DeepSpeedPlugin): + raise TypeError("`deepspeed_plugin` must be a DeepSpeedPlugin object.") + + if deepspeed_plugins is not None: + os.environ["ACCELERATE_USE_DEEPSPEED"] = "true" # use DeepSpeed if plugin is provided + if not is_deepspeed_available(): + raise ImportError("DeepSpeed is not installed => run `pip install deepspeed` or build it from source.") + if is_mlu_available(): + if compare_versions("deepspeed", "<", "0.15.2"): + raise ImportError("DeepSpeed MLU version must be >= 0.15.2. Please update DeepSpeed.") + elif is_musa_available(): + if compare_versions("deepspeed", "<", "0.14.3"): + raise ImportError("DeepSpeed MUSA version must be >= 0.14.3. Please update DeepSpeed.") + elif compare_versions("deepspeed", "<", "0.9.3"): + raise ImportError("DeepSpeed version must be >= 0.9.3. Please update DeepSpeed.") + + self.deepspeed_engine_wrapped = None + + if os.environ.get("ACCELERATE_USE_FSDP", "false").lower() == "true" or isinstance( + fsdp_plugin, FullyShardedDataParallelPlugin + ): + if not is_torch_version(">=", FSDP_PYTORCH_VERSION): + raise ValueError(f"FSDP requires PyTorch >= {FSDP_PYTORCH_VERSION}") + + if fsdp_plugin is None: # init from env variables + fsdp_plugin = ( + FullyShardedDataParallelPlugin() + if os.environ.get("ACCELERATE_USE_FSDP", "false").lower() == "true" + else None + ) + else: + if not isinstance(fsdp_plugin, FullyShardedDataParallelPlugin): + raise TypeError("`fsdp_plugin` must be a FullyShardedDataParallelPlugin object.") + os.environ["ACCELERATE_USE_FSDP"] = "true" # use FSDP if plugin is provided + + if fsdp_plugin is not None and fsdp_plugin.fsdp_version == 2: + if not is_torch_version(">=", FSDP2_PYTORCH_VERSION): + raise ImportError(f"FSDP2 requires PyTorch >= {FSDP2_PYTORCH_VERSION}") + + if megatron_lm_plugin is None: # init from env variables + megatron_lm_plugin = ( + MegatronLMPlugin() if os.environ.get("ACCELERATE_USE_MEGATRON_LM", "false").lower() == "true" else None + ) + else: + if not isinstance(megatron_lm_plugin, MegatronLMPlugin): + raise TypeError("`megatron_lm_plugin` must be a MegatronLMPlugin object.") + os.environ["ACCELERATE_USE_MEGATRON_LM"] = "true" # use MegatronLM if plugin is provided + + if megatron_lm_plugin: + if not is_megatron_lm_available(): + raise ImportError("Megatron is not installed. please build it from source.") + + # Kwargs handlers + self.ddp_handler = None + self.scaler_handler = None + self.init_handler = None + self.fp8_recipe_handler = None + self.ao_recipe_handler = None + self.te_recipe_handler = None + self.msamp_recipe_handler = None + self.autocast_handler = None + self.profile_handler = None + self.has_lomo_optimizer = False + + found_handlers = set() + handler_class_to_attr = { + DistributedDataParallelKwargs: "ddp_handler", + GradScalerKwargs: "scaler_handler", + InitProcessGroupKwargs: "init_handler", + FP8RecipeKwargs: "fp8_recipe_handler", + AutocastKwargs: "autocast_handler", + ProfileKwargs: "profile_handler", + AORecipeKwargs: "ao_recipe_handler", + TERecipeKwargs: "te_recipe_handler", + MSAMPRecipeKwargs: "msamp_recipe_handler", + } + self.has_fp8_handler = False + if kwargs_handlers is not None: + for handler in kwargs_handlers: + assert isinstance(handler, KwargsHandler), ( + f"Unsupported kwargs handler passed: {handler}, must be one that inherits `accelerate.utils.KwargsHandler`." + ) + # Add the handler class to the set of found handlers + if handler.__class__ in found_handlers: + raise ValueError(f"You can only pass one {handler.__class__} in `kwargs_handlers`.") + found_handlers.add(handler.__class__) + handler_attr = handler_class_to_attr[handler.__class__] + setattr(self, handler_attr, handler) + if "recipe_handler" in handler_attr and not self.has_fp8_handler: + self.has_fp8_handler = True + + if parallelism_config is None: + # TODO: Remove after deprecating tp_plugin + if torch_tp_plugin is not None: + parallelism_config = ParallelismConfig(tp_size=torch_tp_plugin.tp_size) + elif os.environ.get("ACCELERATE_USE_PARALLELISM_CONFIG", "false").lower() == "true": + parallelism_config = ParallelismConfig() + + kwargs = self.init_handler.to_kwargs() if self.init_handler is not None else {} + self.state = AcceleratorState( + mixed_precision=mixed_precision, + cpu=cpu, + dynamo_plugin=dynamo_plugin, + deepspeed_plugin=deepspeed_plugins, + fsdp_plugin=fsdp_plugin, + megatron_lm_plugin=megatron_lm_plugin, + parallelism_config=parallelism_config, + _from_accelerator=True, + **kwargs, + ) + + if self.parallelism_config: + self.state.device_mesh = self.parallelism_config.get_device_mesh(self.device.type) + self.parallelism_config._validate_accelerator(self) + + self.fp8_enabled = self.state.mixed_precision == "fp8" or mixed_precision == "fp8" + # Check for automatic FP8 recipe creation + if self.fp8_enabled and not self.has_fp8_handler: + if self.fp8_backend == FP8BackendType.AO: + self.ao_recipe_handler = AORecipeKwargs() + elif self.fp8_backend == FP8BackendType.TE: + self.te_recipe_handler = TERecipeKwargs() + elif self.fp8_backend == FP8BackendType.MSAMP: + self.msamp_recipe_handler = MSAMPRecipeKwargs() + elif self.fp8_backend == FP8BackendType.NO: + # Prioritize AO -> TE -> MSAMP + if is_torchao_available(): + logger.info("Found `torchao` installed, using it for FP8 training.") + self.ao_recipe_handler = AORecipeKwargs() + elif is_transformer_engine_available(): + logger.info("Found `transformer-engine` installed, using it for FP8 training.") + self.te_recipe_handler = TERecipeKwargs() + elif is_msamp_available(): + logger.info("Found `msamp` installed, using it for FP8 training.") + self.msamp_recipe_handler = MSAMPRecipeKwargs() + else: + raise ImportError( + "Tried to train with `fp8` and auto-detect backend, but no FP8-compatible backend was installed. " + "Valid backends are: `torchao`, `transformer-engine`, and `msamp`." + ) + self.has_fp8_handler = True + + self.delayed_fp8_autocast = False + if self.has_fp8_handler: + # We already check if FP8 is available during `self.state` + if not self.fp8_enabled and ( + self.distributed_type not in (DistributedType.FSDP, DistributedType.DEEPSPEED) + ): + raise ValueError("Passing in an FP8 configuration requires setting `mixed_precision='fp8'`.") + self.delayed_fp8_autocast = self.fp8_backend == "TE" and self.distributed_type in ( + DistributedType.MULTI_GPU, + DistributedType.FSDP, + ) + + # TODO: S1ro - this is probably gonna be a problem with other fp8 backends too + if ( + self.fp8_backend == FP8BackendType.AO + and self.state.distributed_type == DistributedType.FSDP + and self.state.fsdp_plugin.cpu_ram_efficient_loading + ): + raise ValueError( + "torchao with FSDP2 and cpu_ram_efficient_loading is not supported, setting `cpu_ram_efficient_loading` to False will fix the issue and work as intended." + ) + + trackers = filter_trackers(log_with, self.logging_dir) + if len(trackers) < 1 and log_with is not None: + warnings.warn(f"`log_with={log_with}` was passed but no supported trackers are currently installed.") + self.log_with = trackers + + if ( + (mixed_precision != "bf16") + and getattr(self.state, "downcast_bfloat", False) + and (self.state.distributedType != DistributedType.XLA) + ): + raise ValueError("Can only use `downcast_bf16` when using `mixed_precision='bf16'` and on a TPU") + + if gradient_accumulation_plugin is not None: + if gradient_accumulation_steps != 1: + raise ValueError( + "You can only pass one of `gradient_accumulation_steps` and `gradient_accumulation_plugin`. Please only pass in the created `GradientAccumulationPlugin` object." + ) + else: + gradient_accumulation_steps = int( + parse_choice_from_env("ACCELERATE_GRADIENT_ACCUMULATION_STEPS", gradient_accumulation_steps) + ) + gradient_accumulation_plugin = GradientAccumulationPlugin(num_steps=gradient_accumulation_steps) + + # If using DeepSpeed, update gradient accumulation steps from the DeepSpeed plugin + self.gradient_state = GradientState( + gradient_accumulation_plugin=gradient_accumulation_plugin, + ) + + self.device_placement = device_placement + if dataloader_config is None: + dataloader_config = DataLoaderConfiguration() + self.dataloader_config = dataloader_config + self.step_scheduler_with_optimizer = step_scheduler_with_optimizer + + # Mixed precision attributes + self.scaler = None + self.native_amp = False + if ( + self.state.mixed_precision == "fp16" + and self.device.type != "cpu" + and self.distributed_type not in (DistributedType.DEEPSPEED, DistributedType.MEGATRON_LM) + ): + self.native_amp = True + supported_device = ("xpu", "cuda", "npu", "xla", "mlu", "musa", "hpu", "sdaa", "mps") + if self.device.type not in supported_device or is_torch_xla_available(check_is_tpu=True): + raise ValueError( + f"fp16 mixed precision requires a device in {supported_device} (not {self.device.type!r})." + ) + if self.device.type == "mps" and not is_torch_version(">=", "2.5.0"): + raise ValueError("fp16 mixed precision with MPS device requires a Pytorch >= 2.5.0") + kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {} + + # FSDP2 doesn't use ShardedGradScaler, don't want to modify `get_grad_scaler`, rather create a simple utility + if self.is_fsdp2: + self.scaler = get_fsdp2_grad_scaler(device=self.device.type, **kwargs) + else: + self.scaler = get_grad_scaler(self.distributed_type, **kwargs) + + elif self.state.mixed_precision == "bf16" and self.distributed_type not in ( + DistributedType.DEEPSPEED, + DistributedType.MEGATRON_LM, + ): + if self.device.type in ["cpu", "xpu", "hpu"]: + self.native_amp = True + else: + self.native_amp = is_bf16_available(True) + if not self.native_amp and not is_torch_xla_available(): + raise ValueError("bf16 mixed precision requires PyTorch >= 1.10 and a supported device.") + if self.native_amp and self.device.type == "mps" and not is_torch_version(">=", "2.6.0"): + raise ValueError("bf16 mixed precision with MPS device requires a Pytorch >= 2.6.0") + + # for DeepSpeed, self.state.mixed_precision is always "bf16", + # see https://github.com/huggingface/accelerate/blob/main/src/accelerate/state.py#L968 and + # https://github.com/huggingface/accelerate/blob/main/src/accelerate/utils/dataclasses.py#L1263. + elif self.fp8_enabled: + # We always enable `native_amp` for FP8 + self.native_amp = True + if self.fp8_backend == FP8BackendType.MSAMP: + if self.distributed_type == DistributedType.FSDP: + raise NotImplementedError( + "`accelerate` + `MS-AMP` + `FSDP` is not supported at this time. " + "Please consider using deepspeed, which is supported." + ) + elif self.distributed_type != DistributedType.DEEPSPEED: + # MS-AMP requires `GradScaler` even with bf16 autocast w/ single GPU or DDP: + self.scaler = get_grad_scaler(**kwargs) + + # Start of internal step tracking + self.step = 0 + + # Internal references to the training objects + self._optimizers = [] + self._models = [] + self._schedulers = [] + self._dataloaders = [] + self._custom_objects = [] + + # Hooks + self._load_model_state_pre_hook = OrderedDict() + self._save_model_state_pre_hook = OrderedDict() + + # RNG Types + self.rng_types = rng_types + if self.rng_types is None: + self.rng_types = ["generator"] + + # Set a flag tensor for early stopping and other breakpoints + self.flag_tensor = None + + check_os_kernel() + + @property + def deepspeed_plugin(self): + """ + Returns the currently active DeepSpeedPlugin. + + If using multiple plugins, the first one will be the active one by default. Manually call + `accelerator.state.select_deepspeed_plugin(key)` to activate a different plugin. + + If deepspeed is not enabled, this will return `None`. + """ + return self.state.deepspeed_plugin + + @property + def use_distributed(self): + """ + Whether the Accelerator is configured for distributed training + """ + return self.state.use_distributed + + @property + def multi_device(self): + return self.use_distributed and self.distributed_type in ( + DistributedType.MULTI_GPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + ) + + @property + def distributed_type(self): + return self.state.distributed_type + + @property + def num_processes(self): + return self.state.num_processes + + @property + def process_index(self): + return self.state.process_index + + @property + def local_process_index(self): + return self.state.local_process_index + + @property + def device(self): + return self.state.device + + @property + def split_batches(self): + return self.dataloader_config.split_batches + + @property + def dispatch_batches(self): + return self.dataloader_config.dispatch_batches + + @property + def even_batches(self): + return self.dataloader_config.even_batches + + @even_batches.setter + def even_batches(self, value: bool): + self.dataloader_config.even_batches = value + + @property + def use_seedable_sampler(self): + return self.dataloader_config.use_seedable_sampler + + @property + def non_blocking(self): + return self.dataloader_config.non_blocking + + @property + def use_stateful_dataloader(self): + if hasattr(self.dataloader_config, "use_stateful_dataloader"): + return self.dataloader_config.use_stateful_dataloader + return False + + @property + def project_dir(self): + return self.project_configuration.project_dir + + @property + def logging_dir(self): + return self.project_configuration.logging_dir + + @property + def save_iteration(self): + return self.project_configuration.iteration + + @property + def is_main_process(self): + """True for one process only.""" + return self.state.is_main_process + + @property + def is_local_main_process(self): + """True for one process per server.""" + return self.state.is_local_main_process + + @property + def is_last_process(self): + return self.process_index == self.num_processes - 1 + + @property + def mixed_precision(self): + return self.state.mixed_precision + + @property + def is_fsdp2(self): + return self.state.is_fsdp2 + + @property + def is_composable_parallelism_enabled(self): + return self.is_fsdp2 + + @property + def parallelism_config(self) -> Union[ParallelismConfig, None]: + return self.state.parallelism_config + + @property + def torch_device_mesh(self): + return self.state.device_mesh + + @property + def should_save_model(self): + if (pc := self.parallelism_config) is None: + # shouldn't even happen + return self.state.is_local_main_process + _non_model_shard_dims = { + pc.dp_replicate_enabled: "dp_replicate", + pc.cp_enabled: "cp", + } + + # return all( + # self.torch_device_mesh[dim].get_local_rank() == 0 for key, dim in non_model_shard_dims.items() if key + # ) + # TODO: S1ro - this is a temporary solution until we figure out why `save_safe_file` is slow when not all processes + return True + + @property + def tensor_parallel_rank(self) -> int: + """ + Returns the local rank for tensor parallelism. If tensor parallelism is configured but not enabled, returns 0 + since all ranks are assumed to be the same. + """ + if self.parallelism_config: + if self.parallelism_config.tp_enabled: + return self.torch_device_mesh.get_local_rank("tp") + return 0 + raise RuntimeError("Tensor parallelism is not configured. Set `parallelism_config` first.") + + @property + def pipeline_parallel_rank(self) -> int: + """ + Pipeline parallelism is not supported yet. + """ + raise NotImplementedError("Pipeline parallelism is currently not supported in Accelerate.") + + @property + def context_parallel_rank(self) -> int: + """ + Context parallelism is not supported yet. + """ + raise NotImplementedError("Context parallelism is currently not supported in Accelerate.") + + @property + def data_parallel_rank(self) -> int: + """ + Returns the local rank for replicate-based data parallelism. If replicate-based data parallelism is configured + but not enabled, returns 0 since all ranks are assumed to be the same. + """ + if self.parallelism_config: + if self.parallelism_config.dp_replicate_enabled: + return self.torch_device_mesh.get_local_rank("dp_replicate") + return 0 + raise RuntimeError("Data parallelism is not configured. Set `parallelism_config` first.") + + @property + def data_parallel_shard_rank(self) -> int: + """ + Returns the local rank for shard-based data parallelism. If shard-based data parallelism is configured but not + enabled, returns 0 since all ranks are assumed to be the same. + """ + if self.parallelism_config: + if self.parallelism_config.dp_shard_enabled: + return self.torch_device_mesh.get_local_rank("dp_shard") + return 0 + raise RuntimeError("Shard-based data parallelism is not configured. Set `parallelism_config` first.") + + @contextmanager + def split_between_processes(self, inputs: list | tuple | dict | torch.Tensor, apply_padding: bool = False): + """ + Splits `input` between `self.num_processes` quickly and can be then used on that process. Useful when doing + distributed inference, such as with different prompts. + + Note that when using a `dict`, all keys need to have the same number of elements. + + Args: + inputs (`list`, `tuple`, `torch.Tensor`, or `dict` of `list`/`tuple`/`torch.Tensor`): + The input to split between processes. + apply_padding (`bool`, `optional`, defaults to `False`): + Whether to apply padding by repeating the last element of the input so that all processes have the same + number of elements. Useful when trying to perform actions such as `Accelerator.gather()` on the outputs + or passing in less inputs than there are processes. If so, just remember to drop the padded elements + afterwards. + + Example: + + ```python + # Assume there are two processes + from accelerate import Accelerator + + accelerator = Accelerator() + with accelerator.split_between_processes(["A", "B", "C"]) as inputs: + print(inputs) + # Process 0 + ["A", "B"] + # Process 1 + ["C"] + + with accelerator.split_between_processes(["A", "B", "C"], apply_padding=True) as inputs: + print(inputs) + # Process 0 + ["A", "B"] + # Process 1 + ["C", "C"] + ``` + """ + with PartialState().split_between_processes(inputs, apply_padding=apply_padding) as inputs: + yield inputs + + def on_main_process(self, function: Callable[..., Any] | None = None): + """ + A decorator that will run the decorated function on the main process only. Can also be called using the + `PartialState` class. + + Args: + function (`Callable`): The function to decorate. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + + + >>> @accelerator.on_main_process + ... def print_something(): + ... print("This will be printed by process 0 only.") + + + >>> print_something() + "This will be printed by process 0 only" + ``` + """ + # For times when the `Accelerator` object itself utilizes this decorator. + if function is None: + if "Accelerator." in self.__qualname__: + function = self + else: + raise ValueError( + "The `on_main_process` decorator must be called with a function on an instantiated `Accelerator` object." + ) + + def _inner(*args, **kwargs): + return PartialState().on_main_process(function)(*args, **kwargs) + + return _inner + + def on_local_main_process(self, function: Callable[..., Any] | None = None): + """ + A decorator that will run the decorated function on the local main process only. Can also be called using the + `PartialState` class. + + Args: + function (`Callable`): The function to decorate. + + Example: + ```python + # Assume we have 2 servers with 4 processes each. + from accelerate import Accelerator + + accelerator = Accelerator() + + + @accelerator.on_local_main_process + def print_something(): + print("This will be printed by process 0 only on each server.") + + + print_something() + # On server 1: + "This will be printed by process 0 only" + # On server 2: + "This will be printed by process 0 only" + ``` + """ + # For times when the `Accelerator` object itself utilizes this decorator. + if function is None: + if "Accelerator." in self.__qualname__: + function = self + else: + raise ValueError( + "The `on_local_main_process` decorator must be called with a function on an instantiated `Accelerator` object." + ) + + def _inner(*args, **kwargs): + return PartialState().on_local_main_process(function)(*args, **kwargs) + + return _inner + + def on_last_process(self, function: Callable[..., Any]): + """ + A decorator that will run the decorated function on the last process only. Can also be called using the + `PartialState` class. + + Args: + function (`Callable`): The function to decorate. + + Example: + ```python + # Assume we have 4 processes. + from accelerate import Accelerator + + accelerator = Accelerator() + + + @accelerator.on_last_process + def print_something(): + print(f"Printed on process {accelerator.process_index}") + + + print_something() + "Printed on process 3" + ``` + """ + # For times when the `Accelerator` object itself utilizes this decorator. + if function is None: + if "Accelerator." in self.__qualname__: + function = self + else: + raise ValueError( + "The `on_last_process` decorator must be called with a function on an instantiated `Accelerator` object." + ) + + def _inner(*args, **kwargs): + return PartialState().on_last_process(function)(*args, **kwargs) + + return _inner + + def on_process(self, function: Callable[..., Any] | None = None, process_index: int | None = None): + """ + A decorator that will run the decorated function on a given process index only. Can also be called using the + `PartialState` class. + + Args: + function (`Callable`, `optional`): + The function to decorate. + process_index (`int`, `optional`): + The index of the process on which to run the function. + + Example: + ```python + # Assume we have 4 processes. + from accelerate import Accelerator + + accelerator = Accelerator() + + + @accelerator.on_process(process_index=2) + def print_something(): + print(f"Printed on process {accelerator.process_index}") + + + print_something() + "Printed on process 2" + ``` + """ + # Initial construction of the decorator. + if (self is not None) and (process_index is not None) and (function is None): + return partial(self.on_process, process_index=process_index) + # For times when the `Accelerator` object itself utilizes this decorator. + if function is None: + if "Accelerator." in self.__qualname__: + function = self + else: + raise ValueError( + "The `on_main_process` decorator must be called with a function on an instantiated `Accelerator` object." + ) + + def _inner(*args, **kwargs): + return PartialState().on_process(function, process_index)(*args, **kwargs) + + return _inner + + def on_local_process(self, function: Callable[..., Any] | None = None, local_process_index: int | None = None): + """ + A decorator that will run the decorated function on a given local process index only. Can also be called using + the `PartialState` class. + + Args: + function (`Callable`, *optional*): + The function to decorate. + local_process_index (`int`, *optional*): + The index of the local process on which to run the function. + + Example: + ```python + # Assume we have 2 servers with 4 processes each. + from accelerate import Accelerator + + accelerator = Accelerator() + + + @accelerator.on_local_process(local_process_index=2) + def print_something(): + print(f"Printed on process {accelerator.local_process_index}") + + + print_something() + # On server 1: + "Printed on process 2" + # On server 2: + "Printed on process 2" + ``` + """ + # Initial construction of the decorator. + if (self is not None) and (local_process_index is not None) and (function is None): + return partial(self.on_local_process, local_process_index=local_process_index) + # For times when the `Accelerator` object itself utilizes this decorator. + if function is None: + if "Accelerator." in self.__qualname__: + function = self + else: + raise ValueError( + "The `on_main_process` decorator must be called with a function on an instantiated `Accelerator` object." + ) + + def _inner(*args, **kwargs): + return PartialState().on_local_process(function, local_process_index)(*args, **kwargs) + + return _inner + + @contextmanager + def main_process_first(self): + """ + Lets the main process go first inside a with block. + + The other processes will enter the with block after the main process exits. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> with accelerator.main_process_first(): + ... # This will be printed first by process 0 then in a seemingly + ... # random order by the other processes. + ... print(f"This will be printed by process {accelerator.process_index}") + ``` + """ + with self.state.main_process_first(): + yield + + @contextmanager + def local_main_process_first(self): + """ + Lets the local main process go inside a with block. + + The other processes will enter the with block after the main process exits. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> with accelerator.local_main_process_first(): + ... # This will be printed first by local process 0 then in a seemingly + ... # random order by the other processes. + ... print(f"This will be printed by process {accelerator.local_process_index}") + ``` + """ + with self.state.local_main_process_first(): + yield + + @contextmanager + def no_sync(self, model): + """ + A context manager to disable gradient synchronizations across DDP processes by calling + `torch.nn.parallel.DistributedDataParallel.no_sync`. + + If `model` is not in DDP, this context manager does nothing + + Args: + model (`torch.nn.Module`): + PyTorch Module that was prepared with `Accelerator.prepare` + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> dataloader, model, optimizer = accelerator.prepare(dataloader, model, optimizer) + >>> input_a = next(iter(dataloader)) + >>> input_b = next(iter(dataloader)) + + >>> with accelerator.no_sync(): + ... outputs = model(input_a) + ... loss = loss_func(outputs) + ... accelerator.backward(loss) + ... # No synchronization across processes, only accumulate gradients + >>> outputs = model(input_b) + >>> accelerator.backward(loss) + >>> # Synchronization across all processes + >>> optimizer.step() + >>> optimizer.zero_grad() + ``` + """ + if self.is_fsdp2: + model.set_requires_gradient_sync(False) + try: + yield + finally: + model.set_requires_gradient_sync(True) + else: + context = contextlib.nullcontext + if self.use_distributed: + if self.distributed_type != DistributedType.DEEPSPEED or self.state.deepspeed_plugin.zero_stage < 2: + context = getattr(model, "no_sync", context) + + with context(): + yield + + @staticmethod + @contextmanager + def trigger_sync_in_backward(model): + """Trigger the sync of the gradients in the next backward pass of the model after multiple forward passes under + `Accelerator.no_sync` (only applicable in multi-GPU scenarios). + + If the script is not launched in distributed mode, this context manager does nothing. + + Args: + model (`torch.nn.Module`): + The model for which to trigger the gradient synchronization. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> dataloader, model, optimizer = accelerator.prepare(dataloader, model, optimizer) + + >>> with accelerator.no_sync(): + ... loss_a = loss_func(model(input_a)) # first forward pass + ... loss_b = loss_func(model(input_b)) # second forward pass + >>> accelerator.backward(loss_a) # No synchronization across processes, only accumulate gradients + >>> with accelerator.trigger_sync_in_backward(model): + ... accelerator.backward(loss_b) # Synchronization across all processes + >>> optimizer.step() + >>> optimizer.zero_grad() + ``` + """ + if not isinstance(model, torch.nn.parallel.DistributedDataParallel): + yield + return + + old_require_backward_grad_sync = model.require_backward_grad_sync + old_require_forward_param_sync = model.require_forward_param_sync + + # EXPERIMENTAL: This will force grad sync during `backward()`, but it is unknown if it breaks other DDP features. + # https://github.com/pytorch/pytorch/blob/e1502c0cdbfd17548c612f25d5a65b1e4b86224d/torch/nn/parallel/distributed.py#L1453-L1466 + model.require_backward_grad_sync = True + model.require_forward_param_sync = True + # https://github.com/pytorch/pytorch/blob/e1502c0cdbfd17548c612f25d5a65b1e4b86224d/torch/csrc/distributed/c10d/reducer.cpp#L1371-L1402 + model.reducer.prepare_for_backward([]) + try: + yield + finally: + model.require_backward_grad_sync = old_require_backward_grad_sync + model.require_forward_param_sync = old_require_forward_param_sync + + def _do_sync(self): + "Sets the right `sync_gradients` context and either resets or increases `self.step`" + if self.gradient_state.sync_with_dataloader and self.gradient_state.end_of_dataloader: + self.step = 0 + self.gradient_state._set_sync_gradients(True) + else: + self.step += 1 + self.gradient_state._set_sync_gradients((self.step % self.gradient_state.num_steps) == 0) + + @property + def sync_gradients(self): + return self.gradient_state.sync_gradients + + @sync_gradients.setter + def sync_gradients(self, sync_gradients): + self.gradient_state.sync_gradients = sync_gradients + + @property + def gradient_accumulation_steps(self): + return self.gradient_state.num_steps + + @gradient_accumulation_steps.setter + def gradient_accumulation_steps(self, gradient_accumulation_steps): + self.gradient_state.plugin_kwargs.update({"num_steps": gradient_accumulation_steps}) + + @contextmanager + def accumulate(self, *models): + """ + A context manager that will lightly wrap around and perform gradient accumulation automatically + + Args: + *models (list of `torch.nn.Module`): + PyTorch Modules that were prepared with `Accelerator.prepare`. Models passed to `accumulate()` will + skip gradient syncing during backward pass in distributed training + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(gradient_accumulation_steps=1) + >>> dataloader, model, optimizer, scheduler = accelerator.prepare(dataloader, model, optimizer, scheduler) + + >>> for input, output in dataloader: + ... with accelerator.accumulate(model): + ... outputs = model(input) + ... loss = loss_func(outputs) + ... loss.backward() + ... optimizer.step() + ... scheduler.step() + ... optimizer.zero_grad() + ``` + """ + self._do_sync() + + allow_gradient_sync = ( + self.sync_gradients # must sync if sync gradients need to complete an optimizer step + or ( + # the no_sync context stops the gradients from reducing during distributed training + # bringing speedup (potentially at some costs). Here, no_sync can be prevented + # by setting sync_each_batch = True. + self.use_distributed # only relevant in distributed settings + and self.gradient_state.plugin_kwargs.get("sync_each_batch", False) + ) + ) + with contextlib.ExitStack() as cm_stack: + for m in models: + cm_stack.enter_context(contextlib.nullcontext() if allow_gradient_sync else self.no_sync(m)) + yield + + @contextmanager + def join_uneven_inputs(self, joinables, even_batches=None): + """ + A context manager that facilitates distributed training or evaluation on uneven inputs, which acts as a wrapper + around `torch.distributed.algorithms.join`. This is useful when the total batch size does not evenly divide the + length of the dataset. + + Args: + joinables (`list[torch.distributed.algorithms.Joinable]`): + A list of models or optimizers that subclass `torch.distributed.algorithms.Joinable`. Most commonly, a + PyTorch Module that was prepared with `Accelerator.prepare` for DistributedDataParallel training. + even_batches (`bool`, *optional*) + If set, this will override the value of `even_batches` set in the `Accelerator`. If it is not provided, + the default `Accelerator` value will be used. + + + + `join_uneven_inputs` is only supported for Distributed Data Parallel training on multiple GPUs. For any other + configuration, this method will have no effect. + + + + + + Overriding `even_batches` will not affect iterable-style data loaders. + + + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(even_batches=True) + >>> ddp_model, optimizer, dataloader = accelerator.prepare(model, optimizer, dataloader) + + >>> with accelerator.join_uneven_inputs([ddp_model], even_batches=False): + ... for input, output in dataloader: + ... outputs = model(input) + ... loss = loss_func(outputs) + ... loss.backward() + ... optimizer.step() + ... optimizer.zero_grad() + ``` + """ + if self.multi_device: + dl_even_batches_values = [] + + if even_batches is not None: + iterable_dl_seen = False + # override value in batch sampler for map-style datasets + for dl_idx, dl in enumerate(self._dataloaders): + if isinstance(dl, DataLoaderDispatcher): + iterable_dl_seen = True + continue + dl_even_batches_values.append((dl_idx, dl.batch_sampler.even_batches)) + dl.batch_sampler.even_batches = even_batches + + if iterable_dl_seen: + warnings.warn( + "Overriding even_batches is only supported for map-style datasets, yet some dataloaders given were iterable" + ) + else: + even_batches = self.even_batches + + enable_join = False if even_batches else True + try: + with Join(joinables, enable=enable_join, throw_on_early_termination=False): + yield + finally: + # reset any batch samplers that have been modified + for dl_idx, even_batches_value in dl_even_batches_values: + self._dataloaders[dl_idx].batch_sampler.even_batches = even_batches_value + else: + # Even when disabled, Join expects models to subclass Joinable, so skip entirely for single process runs + if self.distributed_type != DistributedType.NO: + warnings.warn( + "Joining uneven inputs is only supported for multi-GPU training, as a result `join_uneven_inputs` will have no effect." + ) + + with contextlib.nullcontext(joinables): + yield + + def print(self, *args, **kwargs): + """ + Drop in replacement of `print()` to only print once per server. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> accelerator.print("Hello world!") + ``` + """ + self.state.print(*args, **kwargs) + + def _prepare_one(self, obj, first_pass=False, device_placement=None): + # First pass of preparation: DataLoader, model, optimizer + if first_pass: + if isinstance(obj, torch.utils.data.DataLoader): + return self.prepare_data_loader(obj, device_placement=device_placement) + elif isinstance(obj, torch.nn.Module): + return self.prepare_model(obj, device_placement=device_placement) + elif isinstance(obj, torch.optim.Optimizer): + optimizer = self.prepare_optimizer(obj, device_placement=device_placement) + return optimizer + # Second pass of preparation: LR scheduler (which need the full list of optimizers) + elif isinstance(obj, LRScheduler): + scheduler = self.prepare_scheduler(obj) + return scheduler + # Return the unprocessed object if previous criteria was not met + return obj + + def prepare(self, *args, device_placement=None): + """ + Prepare all objects passed in `args` for distributed training and mixed precision, then return them in the same + order. + + Args: + *args (list of objects): + Any of the following type of objects: + + - `torch.utils.data.DataLoader`: PyTorch Dataloader + - `torch.nn.Module`: PyTorch Module + - `torch.optim.Optimizer`: PyTorch Optimizer + - `torch.optim.lr_scheduler.LRScheduler`: PyTorch LR Scheduler + + device_placement (`list[bool]`, *optional*): + Used to customize whether automatic device placement should be performed for each object passed. Needs + to be a list of the same length as `args`. Not compatible with DeepSpeed or FSDP. + + + + You don't need to prepare a model if you only use it for inference without any kind of mixed precision + + + + Examples: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> # Assume a model, optimizer, data_loader and scheduler are defined + >>> model, optimizer, data_loader, scheduler = accelerator.prepare(model, optimizer, data_loader, scheduler) + ``` + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> # Assume a model, optimizer, data_loader and scheduler are defined + >>> device_placement = [True, True, False, False] + >>> # Will place the first two items passed in automatically to the right device but not the last two. + >>> model, optimizer, data_loader, scheduler = accelerator.prepare( + ... model, optimizer, data_loader, scheduler, device_placement=device_placement + ... ) + ``` + """ + if device_placement is None: + device_placement = [None for _ in args] + elif self.distributed_type in (DistributedType.DEEPSPEED, DistributedType.MEGATRON_LM): + raise ValueError("You can't customize device placements with DeepSpeed or Megatron-LM.") + elif len(device_placement) != len(args): + raise ValueError( + f"`device_placement` should be a list with {len(args)} elements (the number of objects passed)." + ) + + for obj in args: + # TODO: Look at enabling native TP training directly with a proper config + if ( + isinstance(obj, torch.nn.Module) + and self.verify_device_map(obj) + and self.distributed_type != DistributedType.NO + and os.environ.get("ACCELERATE_BYPASS_DEVICE_MAP", "false") != "true" + ): + raise ValueError( + "You can't train a model that has been loaded with `device_map='auto'` in any distributed mode." + " Please rerun your script specifying `--num_processes=1` or by launching with `python {{myscript.py}}`." + ) + + if self.distributed_type == DistributedType.DEEPSPEED: + model_count = 0 + for obj in args: + if isinstance(obj, torch.nn.Module): + model_count += 1 + if model_count > 1: + raise AssertionError( + "You can't use same `Accelerator()` instance with multiple models when using DeepSpeed" + ) + + # On TPUs, putting the model on the XLA device will create new parameters, so the corresponding optimizer will + # have parameters disconnected from the model (so no training :-( ). + # If the model and optimizer have parameters on different devices we raise an error. + if self.distributed_type == DistributedType.XLA: + model_device, optimizer_device = self._get_devices() + if model_device is not None and optimizer_device is not None and model_device != optimizer_device: + raise ValueError( + "The model and the optimizer parameters are not on the same device, which probably means you " + "created an optimizer around your model **before** putting on the device. Make sure the line " + "model.to(device) is before the optimizer creation in your script or remove it entirely and use " + "the flag default value for `device_placement` in your `Accelerator` to let it handle that " + "part for you." + ) + + if self.is_fsdp2: + model_count = 0 + optimizer_count = 0 + for i, obj in enumerate(args): + if isinstance(obj, torch.nn.Module): + model_count += 1 + elif isinstance(obj, torch.optim.Optimizer): + optimizer_count += 1 + + # This needs to be written as such, so that passing other objects other than models/optimizers doesn't raise an error + if (model_count < 1 and optimizer_count > 0) or (model_count > 0 and optimizer_count < 1): + raise ValueError( + "When using FSDP2, a model and optimizer must be passed together to `Accelerator.prepare()`" + " as the optimizer needs to have its parameters modified after the model is converted." + ) + if model_count > 1: + raise ValueError("Only one model is supported when using FSDP2") + + # If we're dealing with device placement, this deals with that by... + tpu_should_fix_optimizer = self.device_placement and self.distributed_type == DistributedType.XLA + + if tpu_should_fix_optimizer: + # 1. grabbing old model parameters + old_named_params = self._get_named_parameters(*args, drop_refs=False) + + if self.parallelism_config and self.parallelism_config.tp_enabled: + args = self._prepare_tp(*args) + for item in args: + if any( + item in container + for container in (self._dataloaders, self._models, self._optimizers, self._schedulers) + ): + item._is_accelerate_prepared = True + + if self.parallelism_config and self.parallelism_config.cp_enabled: + args = self._prepare_cp(*args) + # for megatron-lm, we don't need to prepare TE AO at this moment + if self.distributed_type != DistributedType.MEGATRON_LM: + if self.fp8_backend == FP8BackendType.TE: + args = self._prepare_te(*args) + elif self.fp8_backend == FP8BackendType.AO: + args = self._prepare_ao(*args) + if self.distributed_type == DistributedType.DEEPSPEED: + result = self._prepare_deepspeed(*args) + elif self.distributed_type == DistributedType.MEGATRON_LM: + result = self._prepare_megatron_lm(*args) + elif self.is_fsdp2: + result = self._prepare_fsdp2(*args) + else: + if self.fp8_backend == FP8BackendType.MSAMP: + args, device_placement = self._prepare_msamp(*args, device_placement=device_placement) + result = tuple( + self._prepare_one(obj, first_pass=True, device_placement=d) for obj, d in zip(args, device_placement) + ) + result = tuple(self._prepare_one(obj, device_placement=d) for obj, d in zip(result, device_placement)) + if tpu_should_fix_optimizer: + # 2. grabbing new model parameters + new_named_params = self._get_named_parameters(*result) + # 3. building a map from the first to the second + mapping = {p: new_named_params[n] for n, p in old_named_params.items()} + # 4. using that map to update the parameters of the optimizer + for obj in result: + if isinstance(obj, torch.optim.Optimizer): + obj._switch_parameters(mapping) + + for item in result: + if any( + item in container + for container in (self._dataloaders, self._models, self._optimizers, self._schedulers) + ): + item._is_accelerate_prepared = True + + return result if len(result) > 1 else result[0] + + def _prepare_tp(self, *args): + # First pass: prepare everything except schedulers (first_pass=True) and the model, which is prepared separately + # below + result = [ + self._prepare_one(obj, first_pass=True) if not isinstance(obj, torch.nn.Module) else obj for obj in args + ] + + # Second pass: prepare schedulers + result = [self._prepare_one(obj) if not isinstance(obj, torch.nn.Module) else obj for obj in result] + + for arg in args: + if not isinstance(arg, torch.nn.Module): + continue + model = arg + + from torch.distributed.tensor import DTensor + + if not any(isinstance(p, DTensor) for p in model.parameters()): + logger.warning( + "The model parameters are not sharded by DTensor, we skip the TP preparation. If you are using " + "a PreTrained model it is expected and this warning can be ignored." + ) + return result + + # Now we prepare the model + device_mesh = self.torch_device_mesh + + old_named_params = self._get_named_parameters(*tuple(result), drop_refs=True) + + from torch.distributed.tensor import DTensor + + if self.is_fsdp2: + for arg in result: + if not isinstance(arg, torch.nn.Module): + continue + + from torch.distributed.tensor import Replicate + from transformers.integrations.tensor_parallel import ReplicateParallel + + model: torch.nn.Module = arg + tp_plan = ReplicateParallel + + for name, param in model.named_parameters(): + if isinstance(param, DTensor): + continue + + dp = DTensor.from_local(param, device_mesh=device_mesh["tp"], placements=[Replicate()]) + param_name, param_type = name.rsplit(".", 1) + module_to_tp = model.get_submodule(param_name) + + tp_plan().prepare_module_tp(module_to_tp, device_mesh["tp"]) + if not isinstance(dp, torch.nn.Parameter): + dp = torch.nn.Parameter(dp, requires_grad=param.requires_grad) + setattr(module_to_tp, param_type, dp) + + new_named_params = self._get_named_parameters(*tuple(result), drop_refs=False) + # Build a map from old to new params + mapping = {p: new_named_params[n] for n, p in old_named_params.items()} + + if not mapping: + return result + + def _get_tensor_address(p): + if isinstance(p, DTensor): + return p._local_tensor.data_ptr() + return p.data_ptr() + + for obj in result: + if isinstance(obj, torch.optim.Optimizer): + for param_group in obj.param_groups: + # Each param_group originally maps to model parameters (e.g., from model.parameters()). + # After _prepare_tp(), parameter references are replaced with DTensor instances. + # Therefore, we remap the parameter references to their new DTensor addresses + # so that the optimizer can correctly update the model parameters. + param_group["params"] = [mapping[_get_tensor_address(p)] for p in param_group["params"]] + + return result + + def _prepare_cp(self, *args): + from torch.distributed.tensor.experimental import context_parallel + from torch.distributed.tensor.experimental._attention import set_rotate_method + + cp_comm_strategy = self.parallelism_config.cp_handler.cp_comm_strategy + set_rotate_method(cp_comm_strategy) + + self._cp_context = functools.partial(context_parallel, mesh=self.torch_device_mesh["cp"]) + + for arg in args: + if isinstance(arg, torch.nn.Module): + _attach_context_parallel_hooks(arg) + + return args + + def _prepare_fsdp2(self, *args): + # First pass: prepare everything except schedulers (and model, which is prepared separately below) + result = [ + self._prepare_one(obj, first_pass=True) if not isinstance(obj, torch.nn.Module) else obj for obj in args + ] + + # Second pass: prepare schedulers + result = [self._prepare_one(obj) if not isinstance(obj, torch.nn.Module) else obj for obj in result] + + # Prepare the model + model_index, model = None, None + for i, obj in enumerate(result): + if isinstance(obj, torch.nn.Module): + model_index, model = i, obj + + # Invariant: if we have a model, we also have an optimizer (checked in `prepare`) + if model_index is None: + return tuple(result) + + # Needs to be done first, to make sure AC + fully_shard will work as expected + self.state.fsdp_plugin.set_auto_wrap_policy(model) + + # Apply AC if needed + if self.state.fsdp_plugin.activation_checkpointing: + model = fsdp2_apply_ac(self, model) + + # Apply compile if needed, has to be *after* applying AC + if self.state.dynamo_plugin.backend != DynamoBackend.NO and not is_compiled_module(model): + if self.state.dynamo_plugin.use_regional_compilation: + # Match torchtitan's per-block compile recipe: needed for MoE token-choice dispatch + # (data-dependent dynamic shapes) and to keep the AC + compile boundary consistent + # by skipping replay of forward python side effects in backward. + torch._dynamo.config.capture_scalar_outputs = True + torch._dynamo.config.skip_fwd_side_effects_in_bwd_under_checkpoint = True + model = compile_regions_fsdp2(model, **self.state.dynamo_plugin.to_kwargs()) + else: + model = torch.compile(model, **self.state.dynamo_plugin.to_kwargs()) + + # Get old params and canonicalize - we canonicalize to have the mapping easy + old_named_params = fsdp2_canonicalize_names(self._get_named_parameters(*tuple(result), drop_refs=True)) + + # Swap the optimizer parameters with empty, so `fully_shard` after will not allocate too much memory + from torch.distributed.tensor import DTensor + + for obj in result: + if isinstance(obj, torch.optim.Optimizer): + for param_group in obj.param_groups: + for i, p in enumerate(param_group["params"]): + # We drop a reference to the original param here, so that _move_states_to_device triggers a reallocation + # We reassign the data_ptr to the original param, so that we preserve the mapping to the new ones + param_group["params"][i] = torch.empty(1, dtype=p.dtype, device=p.device) + param_group["params"][i].data_ptr = ( + p._local_tensor.data_ptr() if isinstance(p, DTensor) else p.data_ptr() + ) + + self._models.append(model) + + # Prepare everything FSDP2 related for the model (except AC) + model = fsdp2_prepare_model(self, model) + + # Remove the old model from the list + if len(self._models) > 1 and (self._models[-2] is self._models[-1]): + del self._models[-2] + + # Replace the old model with the new one (shouldn't be needed as everything should be in place) + result[model_index] = model + + # Get new params and canonicalize + new_named_params = fsdp2_canonicalize_names(self._get_named_parameters(*result)) + # Build a map from old to new params and handle missings gracefully + mapping = {} + missing_params = [] + for n, p in old_named_params.items(): + if n in new_named_params: + mapping[p] = new_named_params[n] + else: + missing_params.append(n) + + if missing_params: + # Common tied embedding parameter names + tied_weight_names = ["lm_head.weight", "model.embed_tokens.weight", "transformer.wte.weight"] + if any(name in missing_params for name in tied_weight_names): + raise ValueError( + f"FSDP2 mapping failed (missing: {missing_params}). This is likely due to tied embeddings " + f"(config has tie_word_embeddings=True but checkpoint has separate weights).\n" + f"To fix, try: Set `model.config.tie_word_embeddings = False` after loading the model.\n" + ) + raise KeyError(f"Parameters missing after FSDP2 wrapping: {missing_params}") + + # Update the optimizer parameters + for obj in result: + if isinstance(obj, torch.optim.Optimizer): + fsdp2_switch_optimizer_parameters(obj, mapping) + + return result + + def prepare_model( + self, model: torch.nn.Module, device_placement: bool | None = None, evaluation_mode: bool = False + ): + """ + Prepares a PyTorch model for training in any distributed setup. It is recommended to use + [`Accelerator.prepare`] instead. + + Args: + model (`torch.nn.Module`): + A PyTorch model to prepare. You don't need to prepare a model if it is used only for inference without + any kind of mixed precision + device_placement (`bool`, *optional*): + Whether or not to place the model on the proper device. Will default to `self.device_placement`. + evaluation_mode (`bool`, *optional*, defaults to `False`): + Whether or not to set the model for evaluation only, by just applying mixed precision and + `torch.compile` (if configured in the `Accelerator` object). + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> # Assume a model is defined + >>> model = accelerator.prepare_model(model) + ``` + """ + if device_placement is None: + device_placement = self.device_placement and self.distributed_type != DistributedType.FSDP + + # Ensure we can't double wrap a model + if getattr(model, "_is_accelerate_prepared", False): + if model not in self._models: + self._models.append(model) + return model + + self._models.append(model) + + # TODO: Look at enabling native TP training directly with a proper config + if ( + self.verify_device_map(model) + and self.distributed_type != DistributedType.NO + and os.environ.get("ACCELERATE_BYPASS_DEVICE_MAP", "false") != "true" + ): + raise ValueError( + "You can't train a model that has been loaded with `device_map='auto'` in any distributed mode." + " Please rerun your script specifying `--num_processes=1` or by launching with `python {{myscript.py}}`." + ) + + if self.native_amp: + model._original_forward = model.forward + autocast_context = get_mixed_precision_context_manager(self.native_amp, self.autocast_handler) + # NOTE: MS-AMP adds `__func__` already to `model.forward`, so we should always use `model.forward` + if self.fp8_backend == FP8BackendType.MSAMP or not hasattr(model.forward, "__func__"): + model_forward_func = model.forward + model.forward = convert_outputs_to_fp32(autocast_context(model_forward_func)) + else: + model_forward_func = model.forward.__func__ + new_forward = autocast_context(model_forward_func) + model.forward = MethodType(new_forward, model) + model.forward = MethodType(convert_outputs_to_fp32(model.forward.__func__), model) + + # We prepare TE after, allowing for bf16 autocast to happen first + if self.fp8_backend == FP8BackendType.TE and not self.delayed_fp8_autocast: + model = apply_fp8_autowrap(model, self.te_recipe_handler or self.fp8_recipe_handler) + + if (getattr(model, "is_loaded_in_8bit", False) or getattr(model, "is_loaded_in_4bit", False)) and getattr( + model, "hf_device_map", False + ): + model_devices = set(model.hf_device_map.values()) + if len(model_devices) > 1 and self.distributed_type != DistributedType.NO: + raise ValueError( + "You can't train a model that has been loaded in 8-bit or 4-bit precision on multiple devices in any distributed mode." + " In order to use 8-bit or 4-bit models that have been loaded across multiple GPUs the solution is to use Naive Pipeline Parallelism." + " Therefore you should not specify that you are under any distributed regime in your accelerate config." + ) + elif len(model_devices) == 1: + current_device = list(model_devices)[0] + if isinstance(current_device, torch.device): + current_device_index = current_device.index + elif isinstance(current_device, str): + current_device_index = torch.device(current_device).index + else: + current_device_index = current_device + + current_device_index = int(current_device_index) if current_device_index is not None else None + if self.device.type == "cpu" and is_bitsandbytes_multi_backend_available(): + # bnb with multi-backend supports CPU which don't need to check index. + pass + elif torch.device(self.device.type, current_device_index) != self.device: + # if on the first device (GPU 0) we don't care + if (self.device.index is not None) or (current_device_index != 0): + raise ValueError( + "You can't train a model that has been loaded in 8-bit or 4-bit precision on a different device than the one " + "you're training on. Make sure you loaded the model on the correct device using for example `device_map={'':torch.cuda.current_device()}` or `device_map={'':torch.xpu.current_device()}`" + ) + if ( + ("cpu" in model_devices and not is_bitsandbytes_multi_backend_available()) + or ("cpu" in model_devices and is_xpu_available()) + or "disk" in model_devices + ): + raise ValueError( + "You can't train a model that has been loaded in 8-bit or 4-bit precision with CPU or disk offload. " + "If you want train the 8-bit or 4-bit model in CPU, please install bitsandbytes with multi-backend, see https://huggingface.co/docs/bitsandbytes/main/en/installation#multi-backend" + ) + elif device_placement and not self.verify_device_map(model): + model = model.to(self.device) + if not evaluation_mode: + if self.multi_device and not (self.parallelism_config and self.parallelism_config.tp_enabled): + if model_has_dtensor(model): + raise ValueError( + "Your model contains `DTensor` parameters, which is incompatible with DDP. Maybe you loaded your model with `device_map='auto'`? Specify `device_map='cuda'` or 'xpu' or 'cpu' instead." + ) + if any(p.requires_grad for p in model.parameters()): + kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler is not None else {} + # TODO: Look at enabling native TP training directly with a proper config + if os.environ.get("ACCELERATE_BYPASS_DEVICE_MAP", "false") != "true": + if self.device.type == "hpu": + device_ids, output_device = [self.device.index], self.device.index + else: + device_ids, output_device = [self.local_process_index], self.local_process_index + else: + device_ids, output_device = None, None + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=device_ids, output_device=output_device, **kwargs + ) + if self.ddp_handler is not None: + self.ddp_handler.register_comm_hook(model) + elif self.parallelism_config and self.parallelism_config.tp_enabled: + if not hasattr(model, "tp_size"): + raise NotImplementedError( + "Model should undergo tensor parallel before passing it to accelerate." + "You can use .from_pretrained(..., tp_plan='auto') if the model supports" + ) + if model.tp_size != self.parallelism_config.tp_size: + raise ValueError( + f"tp_size in the plugin {self.parallelism_config.tp_size} should be same as model's tp size {model.tp_size}" + ) + elif self.is_fsdp2: + raise ValueError( + "FSDP2 preparation should be done via `accelerate.prepare()`, as it requires a model and an optimizer." + ) + + elif self.distributed_type == DistributedType.FSDP: + # We need to fix the optimizer *before* sharding the model + from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP + + # Check if the model is already a FSDP model due to `Manual Wrapping` and if so, + # don't wrap it again + # In case the model is already compiled using PyTorch 2.0 and the wrapped model in it + # is a FSDP model, don't wrap it again + is_type_fsdp = isinstance(model, FSDP) or ( + is_compiled_module(model) and isinstance(model._orig_mod, FSDP) + ) + + if not is_type_fsdp: + self.state.fsdp_plugin.set_auto_wrap_policy(model) + fsdp_plugin = self.state.fsdp_plugin + + # need to ensure that params are re-tied after running + # param_init_fn + fsdp_plugin.param_init_fn = ensure_weights_retied( + fsdp_plugin.param_init_fn, + model, + self.device, + ) + + kwargs = { + # We fallback to reshard_after_forward if sharding_strategy is not set. + # We prerfer sharding_strategy to not break the behavior of the existing code. + # Deprecation warning has already been issued in `utils.dataclasses.py` + "sharding_strategy": fsdp_plugin.sharding_strategy or fsdp_plugin.reshard_after_forward, + "cpu_offload": fsdp_plugin.cpu_offload, + "auto_wrap_policy": fsdp_plugin.auto_wrap_policy, + "mixed_precision": fsdp_plugin.mixed_precision_policy, + "sync_module_states": fsdp_plugin.sync_module_states, + "backward_prefetch": fsdp_plugin.backward_prefetch, + "forward_prefetch": fsdp_plugin.forward_prefetch, + "use_orig_params": fsdp_plugin.use_orig_params, + "param_init_fn": fsdp_plugin.param_init_fn, + "ignored_modules": fsdp_plugin.ignored_modules, + "limit_all_gathers": fsdp_plugin.limit_all_gathers, + "device_id": self.device, + } + + if isinstance(kwargs["ignored_modules"], str): + reg = re.compile(kwargs["ignored_modules"]) + ignored = [] + for name, module in model.named_modules(): + if reg.fullmatch(name): + # ensure that the device for these modules is still set correctly + module.to(self.device) + ignored.append(module) + kwargs["ignored_modules"] = ignored + + model = FSDP(model, **kwargs) + if fsdp_plugin.activation_checkpointing: + from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import ( + CheckpointImpl, + apply_activation_checkpointing, + checkpoint_wrapper, + ) + + apply_activation_checkpointing( + model, + checkpoint_wrapper_fn=functools.partial( + checkpoint_wrapper, + checkpoint_impl=CheckpointImpl.NO_REENTRANT, + ), + auto_wrap_policy=fsdp_plugin.auto_wrap_policy, + ) + + # In the event the model had been loaded in low precision, but + # mixed precision had also been activated, then we follow DeepSpeed's + # strategy to hold the parameters in full precision. + # - assume that trainer.args.bf16 and trainer.args.fp16 are already checked against + # fsdp_plugin.mixed_precision_policy. + # - NOTE: we do not check the mixed_precision attribute on the FSDP root wrapper. + # * this attribute will always set by init_utils.init_core_state so its always not None. + # * mixed_precision.param_dtype only regards _fwd_bwd_param_dtype + # * if model is loaded in 16bit, and even if mixed_precision.param_dtype is None, + # we still want to upcast the flat_param. + if self.mixed_precision != "no": # if mixed precision is set + upcasted_log = [] + for module in FSDP.fsdp_modules(model): + # Referencing DeepSpeed Zero3 + # - in Init, params are converted to 16bit while partitioning. + # - in accelerator.prepare, deepspeed.initialize is called to: + # * creates the DeepSpeedEngine. + # * since zero_optimization() is True , calls engine._configure_zero_optimizer. + # + # Inside the DeepSpeed Zero3 optimizer configuration, which initializes + # DeepSpeedZeroOptimizer_Stage3, during which: + # * trainable_param_groups are obtained from the attached optimizer + # (already partitioned in 16bit). + # * then _setup_for_real_optimizer -> _create_fp32_partitions + # which performs the fp32 upcasting. + + # To mimic DeepSeepds's casting in FSDP, we look at the (single) FlatParameter held + # within an FSDP wrapper. This FlatParameter will be seen by the optimizer. + # - even though there is a torch.device('meta') guard below, we + # expect _init_utils._init_param_handle_from_module to already + # sync the parameter. + + if not module._has_params: + continue # skip if FSDP module not managing parameters + param = module._flat_param + if ( + param.dtype != torch.float32 + and param.device != torch.device("meta") + and param.requires_grad + ): + # keep log of names_params that was upcasted + # NOTE: resorted to this because warnings.simplefilter("once") is somehow not working + name_param_log = (module.module.__class__.__name__, ", ".join(module._flat_param._fqns)) + if name_param_log not in upcasted_log: + upcasted_log.append(name_param_log) + + # this works because of FSDP's _runtime_utils.lazy_init. + # Have to be careful not to call anything before this that + # triggers lazy_init (e.g., _is_fsdp_root). + param.data = param.data.to(torch.float32) # upcasting + module._handle._orig_param_dtype = torch.float32 # update + + # report the warnings + # some messages can be quite repetitive, especially when reporting about layers that have identical architecture. + if self.is_main_process: + for name_log, param_log in upcasted_log: + warnings.warn( + f"Upcasted low precision parameters in {name_log} because mixed precision turned on in FSDP. " + f"Affects: {param_log}." + ) + + if len(upcasted_log) > 0: + warnings.warn( + "FSDP upcast of low precision parameters may affect the precision of model checkpoints." + ) + + # if the previous and current models are same, delete the previous one + if len(self._models) > 1 and (self._models[-2] is self._models[-1]): + del self._models[-2] + self._models[-1] = model + elif self.distributed_type == DistributedType.MULTI_CPU: + kwargs = self.ddp_handler.to_kwargs() if self.ddp_handler else {} + model = torch.nn.parallel.DistributedDataParallel(model, **kwargs) + if self.ddp_handler is not None: + self.ddp_handler.register_comm_hook(model) + elif self.distributed_type == DistributedType.XLA and self.state.fork_launched: + model = xmp.MpModelWrapper(model).to(self.device) + # Now we can apply the FP8 autocast + if self.fp8_backend == FP8BackendType.TE and self.delayed_fp8_autocast: + model = apply_fp8_autowrap(model, self.te_recipe_handler or self.fp8_recipe_handler) + # torch.compile should be called last and only if the model isn't already compiled + if self.state.dynamo_plugin.backend != DynamoBackend.NO and not is_compiled_module(model): + if self.state.dynamo_plugin.use_regional_compilation: + model = compile_regions(model, **self.state.dynamo_plugin.to_kwargs()) + else: + model = torch.compile(model, **self.state.dynamo_plugin.to_kwargs()) + model._is_accelerate_prepared = True + return model + + def _prepare_ao(self, *args): + if not is_torchao_available(): + raise ImportError( + "`torchao` was not found on your system or is too old of a version. Please ensure that `torchao >= 0.6.1` is installed" + ) + + if self.is_fsdp2: + models = [x for x in args if isinstance(x, torch.nn.Module)] + optimizers = [x for x in args if isinstance(x, torch.optim.Optimizer)] + for arg in args: + if isinstance(arg, torch.nn.Module): + convert_model_to_fp8_ao( + arg, + config=self.ao_recipe_handler.config, + module_filter_func=self.ao_recipe_handler.module_filter_func, + ) + + # Invariant: with FSDP2, optimizer is always passed to `prepare()` together with model + # We only precompute scales if float8 all gather is enabled, possibly can add a flag for this later + if self.is_fsdp2 and len(optimizers) > 0 and self.ao_recipe_handler.config.enable_fsdp_float8_all_gather: + from torchao.float8 import precompute_float8_dynamic_scale_for_fsdp + + optimizers[0].register_step_post_hook( + lambda *args, **kwargs: precompute_float8_dynamic_scale_for_fsdp(models[0]) + ) + + return args + + def _prepare_te(self, *args): + if not is_transformer_engine_available(): + raise ImportError( + "`transformer_engine` was not found on your system. Please ensure that `transformer_engine` is installed" + ) + model, optimizer = None, None + num_models, num_optimizers = 0, 0 + result = [obj for obj in args] + for obj in result: + if isinstance(obj, torch.nn.Module): + model = obj + num_models += 1 + elif isinstance(obj, (torch.optim.Optimizer)): + optimizer = obj + num_optimizers += 1 + if optimizer is None and model is None: + return result + elif optimizer is None or model is None: + raise ValueError( + "You must pass a model and an optimizer together to `accelerate.prepare()` when using TransformerEngine." + ) + elif num_models > 1 or num_optimizers > 1: + raise ValueError( + f"You can't use multiple models ({num_models}) or optimizers {num_optimizers} with TransformerEngine." + ) + old_named_params = self._get_named_parameters(model) + with torch.no_grad(): + convert_model(model) + new_named_params = self._get_named_parameters(model) + mapping = {p: new_named_params[n] for n, p in old_named_params.items()} + # We need to switch the optimizer params to the new params *after* the model is wrapped in FSDP + for param_group in optimizer.param_groups: + param_group["params"] = [mapping[p] for p in param_group["params"]] + + return result + + def _prepare_deepspeed(self, *args): + import deepspeed + + ds_initialize = deepspeed.initialize + if self.fp8_backend == FP8BackendType.MSAMP: + # MS-AMP requires DeepSpeed patches + from msamp import deepspeed as msamp_deepspeed + + ds_initialize = msamp_deepspeed.initialize + + deepspeed_plugin = self.deepspeed_plugin + + is_dataloader_present = any(isinstance(obj, torch.utils.data.DataLoader) for obj in args) + tp_size = deepspeed_plugin.deepspeed_config.get("tensor_parallel", {}).get("autotp_size", 0) + + sp_backend = self.parallelism_config.sp_backend if self.parallelism_config else None + sp_size = self.parallelism_config.sp_size if self.parallelism_config else 1 + sp_handler = self.parallelism_config.sp_handler if self.parallelism_config else None + + if tp_size > 1: + if not compare_versions("deepspeed", ">=", "0.16.4"): + raise ImportError( + "Deepspeed TP requires deepspeed >= 0.16.4, Please update DeepSpeed via `pip install deepspeed -U`." + ) + if not is_torch_version(">=", "2.2.0"): + raise ImportError( + "Tried to use TP, but `torch.distributed.device_mesh` requires PyTorch >= 2.2.0. Please upgrade your PyTorch version" + ) + from torch.distributed.device_mesh import init_device_mesh + + mesh_dim_name = "tp" + self.state.ds_device_mesh = init_device_mesh(self.device.type, (tp_size,), mesh_dim_names=(mesh_dim_name,)) + + result = [ + self._prepare_one(obj, first_pass=True) if isinstance(obj, torch.utils.data.DataLoader) else obj + for obj in args + ] + + if deepspeed_plugin.is_auto("train_micro_batch_size_per_gpu"): + if is_dataloader_present: + batch_sizes = [obj.batch_size for obj in args if hasattr(obj, "batch_size")] + if any(bs is None for bs in batch_sizes): + raise ValueError( + "At least one of the dataloaders passed to `accelerate.prepare()` has `None` as batch size. " + "Please set an integer value in `train_micro_batch_size_per_gpu` in the deepspeed config file " + "or assign integer value to `AcceleratorState().deepspeed_plugin.deepspeed_config['train_micro_batch_size_per_gpu']`." + ) + if self.split_batches: + batch_sizes = [batch_size // self.num_processes for batch_size in batch_sizes] + + batch_size_per_device = min(batch_sizes) if deepspeed_plugin.is_train_batch_min else max(batch_sizes) + if len(batch_sizes) > 1: + logger.info( + "Since you passed both train and evaluation dataloader, `is_train_batch_min` (here " + f"{deepspeed_plugin.is_train_batch_min} will decide the `train_batch_size` ({batch_size_per_device})." + ) + else: + raise ValueError( + "When using DeepSpeed, `accelerate.prepare()` requires you to pass at least one of training or evaluation dataloaders " + "with `batch_size` attribute returning an integer value " + "or alternatively set an integer value in `train_micro_batch_size_per_gpu` in the deepspeed config file " + "or assign integer value to `AcceleratorState().deepspeed_plugin.deepspeed_config['train_micro_batch_size_per_gpu']`." + ) + else: + batch_size_per_device = deepspeed_plugin.get_value("train_micro_batch_size_per_gpu") + + # handle `gradient_accumulation_steps` when the value is `auto` + deepspeed_plugin.fill_match( + "gradient_accumulation_steps", + must_match=False, + gradient_accumulation_steps=self.gradient_accumulation_steps, + ) + + deepspeed_gradient_accumulation_steps = deepspeed_plugin.get_value("gradient_accumulation_steps") + # update gradient_accumulation_steps if there is a mismatch + if deepspeed_gradient_accumulation_steps != self.gradient_accumulation_steps: + logger.warning( + f"Gradient accumulation steps mismatch: GradientAccumulationPlugin has {self.gradient_accumulation_steps}, " + f"DeepSpeed config has {deepspeed_gradient_accumulation_steps}. Using DeepSpeed's value." + ) + self.gradient_accumulation_steps = deepspeed_gradient_accumulation_steps + + config_kwargs = { + "gradient_clipping": 1.0, + "zero_optimization.stage3_gather_16bit_weights_on_model_save": False, + } + # This block is skipped when preparing just a model and DL is absent from current call's args + if batch_size_per_device is not None: + config_kwargs["train_micro_batch_size_per_gpu"] = batch_size_per_device + config_kwargs["train_batch_size"] = ( + batch_size_per_device + * deepspeed_plugin.get_value("gradient_accumulation_steps") + * self.num_processes + // sp_size + ) + + model = None + optimizer = None + scheduler = None + for obj in result: + if isinstance(obj, torch.nn.Module): + model = obj + elif isinstance(obj, (torch.optim.Optimizer, DummyOptim)): + optimizer = obj + elif (isinstance(obj, (LRScheduler, DummyScheduler))) or ( + type(obj).__name__ in deepspeed.runtime.lr_schedules.VALID_LR_SCHEDULES + ): + scheduler = obj + + if optimizer is not None: + if "optimizer" in deepspeed_plugin.deepspeed_config and not isinstance(optimizer, (DummyOptim)): + raise ValueError( + "You cannot specify an optimizer in the config file and in the code at the same time. " + "Please remove the optimizer from the config file or " + "create `accelerate.utils.DummyOptim` in the code." + ) + elif "optimizer" not in deepspeed_plugin.deepspeed_config and isinstance(optimizer, (DummyOptim)): + raise ValueError( + "You cannot create a `DummyOptim` without specifying an optimizer in the config file." + ) + + if isinstance(optimizer, (torch.optim.Optimizer)): + deepspeed_plugin.deepspeed_config["zero_allow_untested_optimizer"] = True + + if scheduler is not None: + if "scheduler" in deepspeed_plugin.deepspeed_config and not isinstance(scheduler, (DummyScheduler)): + raise ValueError( + "You cannot specify a scheduler in the config file and in the code at the same time. " + "Please remove the scheduler from the config file or " + "create `accelerate.utils.DummyScheduler` in the code." + ) + elif ( + "scheduler" not in deepspeed_plugin.deepspeed_config + and isinstance(scheduler, (DummyScheduler)) + and scheduler.lr_scheduler_callable is None + ): + raise ValueError( + "Either specify a scheduler in the config file or " + "pass in the `lr_scheduler_callable` parameter when using `accelerate.utils.DummyScheduler`." + ) + + if optimizer is not None and scheduler is not None: + if isinstance(optimizer, (DummyOptim)) and not isinstance(scheduler, (DummyScheduler)): + raise ValueError( + "You can only specify `accelerate.utils.DummyScheduler` in the code when using " + "`accelerate.utils.DummyOptim`." + ) + + if model is not None: + # If we are using FP8, we need to apply the autowrap now + if self.fp8_backend == FP8BackendType.TE: + model = apply_fp8_autowrap(model, self.fp8_recipe_handler) + # if the model is an MOE, set the appropriate MOE layers as leaf Z3 modules + deepspeed_plugin.set_moe_leaf_modules(model) + # deal with config keys that use `auto` value and rely on model's hidden_size + hidden_size_based_keys = [ + "zero_optimization.reduce_bucket_size", + "zero_optimization.stage3_prefetch_bucket_size", + "zero_optimization.stage3_param_persistence_threshold", + ] + hidden_size_auto_keys = [x for x in hidden_size_based_keys if deepspeed_plugin.is_auto(x)] + if len(hidden_size_auto_keys) > 0: + reasoning = ( + "therefore it's not possible to automatically fill out the following `auto` entries " + + f"in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing " + + "`auto` values for these keys with an integer value of your choice." + ) + if not hasattr(model, "config"): + raise ValueError("Can't find `model.config` entry, " + reasoning) + + if hasattr(model.config, "hidden_size"): + hidden_size = model.config.hidden_size + elif hasattr(model.config, "hidden_sizes"): + # if there are many hidden sizes pick the largest one + hidden_size = max(model.config.hidden_sizes) + else: + raise ValueError( + "Can find neither `model.config.hidden_size` nor `model.config.hidden_sizes`, " + reasoning + ) + + config_kwargs.update( + { + "zero_optimization.reduce_bucket_size": hidden_size * hidden_size, + "zero_optimization.stage3_prefetch_bucket_size": int(0.9 * hidden_size * hidden_size), + "zero_optimization.stage3_param_persistence_threshold": 10 * hidden_size, + } + ) + + if isinstance(optimizer, (DummyOptim)): + config_kwargs.update( + {"optimizer.params.lr": optimizer.lr, "optimizer.params.weight_decay": optimizer.weight_decay} + ) + if isinstance(scheduler, (DummyScheduler)) and scheduler.lr_scheduler_callable is None: + max_lr = ( + getattr(scheduler.optimizer, "lr", None) + if getattr(scheduler.optimizer, "defaults", None) is None + else scheduler.optimizer.defaults["lr"] + ) + config_kwargs.update( + { + "scheduler.params.warmup_min_lr": 0, + "scheduler.params.warmup_max_lr": max_lr, + "scheduler.params.warmup_num_steps": scheduler.warmup_num_steps, + } + ) + if scheduler.total_num_steps is not None: + config_kwargs["scheduler.params.total_num_steps"] = ( + math.ceil(scheduler.total_num_steps / self.num_processes) + if not self.split_batches + else scheduler.total_num_steps + ) + + deepspeed_plugin.deepspeed_config_process(must_match=False, **config_kwargs) + self.deepspeed_config = deepspeed_plugin.deepspeed_config + + # note: batch_size derivation is all over the map, especially in HF Trainer, so try to fix it at the last moment if needed + pc = self.parallelism_config + if pc is not None and pc.sp_backend == "deepspeed" and pc.sp_size > 1: + self.deepspeed_config["train_batch_size"] = ( + self.deepspeed_config["train_micro_batch_size_per_gpu"] + * self.deepspeed_config["gradient_accumulation_steps"] + * pc.data_parallel_size + ) + + kwargs = dict(model=model, config_params=self.deepspeed_config) + if optimizer is not None: + if isinstance(optimizer, (DummyOptim)): + kwargs["model_parameters"] = optimizer.params + if isinstance(scheduler, (DummyScheduler)) and scheduler.lr_scheduler_callable is not None: + kwargs["lr_scheduler"] = scheduler.lr_scheduler_callable + else: + if self.deepspeed_config["zero_optimization"].get("offload_optimizer", {}).get( + "device", "none" + ) != "none" and self.deepspeed_config.get("zero_force_ds_cpu_optimizer", True): + if self.device.type == "hpu" and os.environ.get("PT_HPU_LAZY_MODE", "1") == "1": + raise ValueError( + "You can't use an Offload Optimizer with HPU in Lazy Mode. " + "Please set the environment variable `PT_HPU_LAZY_MODE` to `0`." + ) + + optimizer = map_pytorch_optim_to_deepspeed(optimizer) + kwargs["optimizer"] = optimizer + if scheduler is not None: + if type(scheduler).__name__ in deepspeed.runtime.lr_schedules.VALID_LR_SCHEDULES: + kwargs["lr_scheduler"] = scheduler + + if self.device.type == "hpu": + # This env variable is initialized here to make sure it is set to "true" + # It should be done by the launcher but it does not work for multi-node runs + os.environ["DEEPSPEED_USE_HPU"] = "true" + + mpu = None + if sp_size > 1: + if sp_backend != "deepspeed": + raise ValueError( + f"In order to use the configured {sp_size=} with DeepSpeed, you need to configure sp_backend='deepspeed', yet you configured it to be {sp_backend=}." + ) + + ver_min_required = "0.18.2" + if not compare_versions("deepspeed", ">=", ver_min_required): + raise ImportError( + f"Deepspeed ALST/Ulysses requires deepspeed>={ver_min_required}. Please update DeepSpeed via `pip install deepspeed -U`." + ) + + from deepspeed.runtime.sequence_parallel.ulysses_sp import ( + UlyssesSPAttentionHF, + UlyssesSPDataLoaderAdapter, + ) + + if not hasattr(model, "config"): + raise ValueError( + "UlyssesSPAttentionHF currently works with HF Transformers and expects the model object to have a config attribute but this model doesn't have one." + ) + + kwagrs = {} + signature = inspect.signature(UlyssesSPAttentionHF.register_with_transformers) + if "disable_in_eval" in signature.parameters.keys(): + kwagrs["disable_in_eval"] = True + + mpu = UlyssesSPAttentionHF.register_with_transformers( + model_name_or_path=model, + sequence_parallel_size=sp_size, + seq_length=sp_handler.sp_seq_length, + seq_length_is_variable=sp_handler.sp_seq_length_is_variable, + core_attn_implementation=sp_handler.sp_attn_implementation, + micro_batch_size=batch_size_per_device, + **kwagrs, + ) + kwargs["mpu"] = mpu + + for i in range(len(result)): + if isinstance(result[i], torch.utils.data.DataLoader): + if sp_size > 1: + # note that in case dataloader was prepared apart from model (for the external accelerator.prepare call) you'd need to call deepspeed_ulysses_dl_adapter after prepare(model) (see HF Trainer as the use-case) + sp_group = mpu.get_sequence_parallel_group() + sp_world_size = mpu.get_sequence_parallel_world_size() + sp_rank = mpu.get_sequence_parallel_rank() + result[i] = UlyssesSPDataLoaderAdapter( + result[i], + sp_rank=sp_rank, + sp_group=sp_group, + sp_world_size=sp_world_size, + device=self.device, # model.device, + ) + + engine, optimizer, _, lr_scheduler = ds_initialize(**kwargs) + + if compare_versions("deepspeed", ">=", "0.14.4") and self.state.dynamo_plugin.backend != DynamoBackend.NO: + compile_kwargs = self.state.dynamo_plugin.to_kwargs() + if self.state.dynamo_plugin.use_regional_compilation: + compile_regions_deepspeed(engine.module, **compile_kwargs) + else: + engine.compile(backend=compile_kwargs.pop("backend"), compile_kwargs=compile_kwargs) + if optimizer is not None: + optimizer = DeepSpeedOptimizerWrapper(optimizer) + if scheduler is not None: + if lr_scheduler is None: + scheduler = AcceleratedScheduler( + scheduler, + optimizer, + step_with_optimizer=self.step_scheduler_with_optimizer, + split_batches=self.split_batches, + ) + else: + scheduler = DeepSpeedSchedulerWrapper(lr_scheduler, optimizer) + + for i in range(len(result)): + if isinstance(result[i], torch.nn.Module): + result[i] = engine + elif isinstance(result[i], (torch.optim.Optimizer, DummyOptim)): + result[i] = optimizer + elif (isinstance(result[i], (LRScheduler, DummyScheduler))) or ( + type(result[i]).__name__ in deepspeed.runtime.lr_schedules.VALID_LR_SCHEDULES + ): + result[i] = scheduler + + # pointing for deepspeed_engine_wrapped.backward() + if self.deepspeed_engine_wrapped is None: + self.deepspeed_engine_wrapped = DeepSpeedEngineWrapper(engine) + else: + logger.warning( + "A wrapped DeepSpeed engine reference is currently tied for this `Accelerator()` instance. " + "If you want to call `accelerator.backward()` referencing a new model/engine, " + "please create a separate `Accelerator()` instance and call `accelerator.prepare()` on it." + ) + self._models.append(engine) + if optimizer is not None: + self._optimizers.append(optimizer) + if scheduler is not None: + self._schedulers.append(scheduler) + return tuple(result) + + def deepspeed_ulysses_dl_adapter(self, dl, model): + """this is normally called as part of `prepare` but when dataloader was prepared apart from model (for the external accelerator.prepare call) this additional call needs to be made after prepare(model) (see HF Trainer as the use-case)""" + sp_size = self.parallelism_config.sp_size if self.parallelism_config else 1 + if sp_size == 1: + return dl + from deepspeed.runtime.sequence_parallel.ulysses_sp import UlyssesSPDataLoaderAdapter + from deepspeed.utils import groups + + sp_group = groups._get_sequence_parallel_group() + sp_world_size = groups._get_sequence_parallel_world_size() + sp_rank = groups._get_sequence_parallel_rank() + dl = UlyssesSPDataLoaderAdapter( + dl, + sp_rank=sp_rank, + sp_group=sp_group, + sp_world_size=sp_world_size, + device=model.device, + ) + return dl + + def _prepare_megatron_lm(self, *args): + megatron_lm_plugin = self.state.megatron_lm_plugin + micro_batch_size = None + if not megatron_lm_plugin.megatron_dataset_flag: + batch_sizes = [obj.batch_size for obj in args if hasattr(obj, "batch_size")] + if len(batch_sizes) == 0: + raise ValueError( + "You must specify a training or evaluation dataloader in `accelerate.prepare()` when using Megatron-LM." + ) + + micro_batch_size = min(batch_sizes) if megatron_lm_plugin.is_train_batch_min else max(batch_sizes) + if len(batch_sizes) > 1: + logger.info( + "Since you passed both train and evaluation dataloader, `is_train_batch_min` (here " + f"{megatron_lm_plugin.is_train_batch_min} will decide the `train_batch_size` ({micro_batch_size})." + ) + else: + for obj in args: + if isinstance(obj, MegatronLMDummyDataLoader): + micro_batch_size = obj.dataset_args["micro_batch_size"] + break + if micro_batch_size is not None: + dp_degree = self.num_processes // (megatron_lm_plugin.tp_degree * megatron_lm_plugin.pp_degree) + megatron_lm_plugin.set_training_args(micro_batch_size, dp_degree) + else: + raise ValueError( + "When you do not pass the dataloader parameter, the `data_parallel_size`, " + "`micro_batch_size`, and `global_batch_size` megatron parameters will not be updated." + ) + model = None + optimizer = None + scheduler = None + batch_data = None + for obj in args: + if isinstance(obj, torch.utils.data.DataLoader) and batch_data is None: + batch_data = next(iter(obj)) + elif isinstance(obj, torch.nn.Module): + model = obj + elif isinstance(obj, (torch.optim.Optimizer)): + optimizer = obj + elif isinstance(obj, (LRScheduler, MegatronLMDummyScheduler)): + scheduler = obj + + if model is not None: + megatron_lm_plugin.set_network_size_args(model, batch_data) + if optimizer is not None: + megatron_lm_plugin.set_optimizer_type(optimizer) + if scheduler is not None: + if not isinstance(scheduler, MegatronLMDummyScheduler): + raise ValueError( + "You can't use a custom scheduler with Megatron-LM. Please use the `accelerate.utils.MegatronLMDummyScheduler` instead." + ) + megatron_lm_plugin.set_scheduler_args(scheduler) + + # initialize megatron-lm + megatron_lm_initialize(self, args_defaults=megatron_lm_plugin.megatron_lm_default_args) + + (model, optimizer, scheduler) = megatron_lm_prepare_model_optimizer_scheduler(self) + self.wait_for_everyone() + + counter = 0 + result = [] + for obj in args: + if isinstance(obj, torch.utils.data.DataLoader): + result.append(megatron_lm_prepare_data_loader(self, obj)) + counter += 1 + elif isinstance(obj, MegatronLMDummyDataLoader): + if counter == 0: + obj.set_megatron_data_args() + dataloaders = megatron_lm_prepare_data_loader(self, obj) + result.append(dataloaders[counter]) + counter += 1 + else: + result.append(obj) + + if model is not None: + model = MegatronEngine(self, model, optimizer, scheduler) + if optimizer is not None: + optimizer = MegatronLMOptimizerWrapper(optimizer) + if scheduler is not None: + scheduler = MegatronLMSchedulerWrapper(scheduler, optimizer) + + for i in range(len(result)): + if isinstance(result[i], torch.nn.Module): + result[i] = model + elif isinstance(result[i], torch.optim.Optimizer): + result[i] = optimizer + elif isinstance(result[i], MegatronLMDummyScheduler): + result[i] = scheduler + + if model is not None: + self._models.append(model) + if len(self._models) > 1: + raise AssertionError( + "You can't use same `Accelerator()` instance with multiple models when using Megatron-LM" + ) + if optimizer is not None: + self._optimizers.append(optimizer) + if scheduler is not None: + self._schedulers.append(scheduler) + + return tuple(result) + + def _prepare_device_mesh(self): + """ + Prepare the device mesh for distributed training. The dataloader will determine how to load data based on the + device mesh. + """ + if self.distributed_type == DistributedType.DEEPSPEED and hasattr(self.state, "ds_device_mesh"): + return self.state.ds_device_mesh + else: + return self.torch_device_mesh + + def _prepare_msamp(self, *args, device_placement): + warnings.warn( + "MS-AMP is deprecated and will be removed in a future version of Accelerate. " + "Please use `'te'` (Transformer Engine) or `'torchao'` as the backend for FP8 " + "mixed precision training instead.", + FutureWarning, + ) + if not is_msamp_available(): + raise ImportError( + "MS-AMP was not found on your system. Please ensure that MS-AMP is available " + " or choose `'te'` as the backend for FP8 mixed precision training." + ) + # We've already checked for FSDP + MS-AMP during `__init__` + import msamp + + model, optimizer = None, None + optimizer_index = None + num_models, num_optimizers = 0, 0 + result = [obj for obj in args] + for i, obj in enumerate(result): + if isinstance(obj, torch.nn.Module): + model = obj + num_models += 1 + elif isinstance(obj, (torch.optim.Optimizer)): + optimizer = obj + optimizer_index = i + num_optimizers += 1 + # DataLoader/Scheduler case + if optimizer is None and model is None: + return result, device_placement + elif optimizer is None or model is None: + raise ValueError( + "You must pass a model and an optimizer together to `accelerate.prepare()` when using MS-AMP." + ) + elif num_models > 1 or num_optimizers > 1: + raise ValueError( + f"You can't use multiple models ({num_models}) or optimizers {num_optimizers} with MS-AMP." + ) + else: + # DEPRECATE @ 2.0 + if self.fp8_recipe_handler is not None: + opt_level = self.fp8_recipe_handler.opt_level + else: + opt_level = self.msamp_recipe_handler.opt_level + model, optimizer = msamp.initialize(model, optimizer, opt_level=opt_level) + for i in range(len(result)): + if isinstance(result[i], torch.nn.Module): + result[i] = model + elif isinstance(result[i], (torch.optim.Optimizer)): + result[i] = optimizer + if optimizer_index is not None: + # NOTE: MS-AMP moves the optimizer, but *not* the model to the right device + device_placement[optimizer_index] = False + return tuple(result), device_placement + + def prepare_data_loader( + self, data_loader: torch.utils.data.DataLoader, device_placement=None, slice_fn_for_dispatch=None + ): + """ + Prepares a PyTorch DataLoader for training in any distributed setup. It is recommended to use + [`Accelerator.prepare`] instead. + + Args: + data_loader (`torch.utils.data.DataLoader`): + A vanilla PyTorch DataLoader to prepare + device_placement (`bool`, *optional*): + Whether or not to place the batches on the proper device in the prepared dataloader. Will default to + `self.device_placement`. + slice_fn_for_dispatch (`Callable`, *optional*`): + If passed, this function will be used to slice tensors across `num_processes`. Will default to + [`~utils.slice_tensors`]. This argument is used only when `dispatch_batches` is set to `True` and will + be ignored otherwise. + + Example: + + ```python + >>> import torch + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> data_loader = torch.utils.data.DataLoader(...) + >>> data_loader = accelerator.prepare_data_loader(data_loader, device_placement=True) + ``` + """ + # Ensure we can't double wrap a DataLoader due to `find_batch_size` + if getattr(data_loader, "_is_accelerate_prepared", False): + if data_loader not in self._dataloaders: + self._dataloaders.append(data_loader) + return data_loader + if device_placement is None: + device_placement = self.device_placement if self.distributed_type != DistributedType.XLA else False + + device_mesh = self._prepare_device_mesh() + + prepared_data_loader = prepare_data_loader( + data_loader, + self.device, + num_processes=self.num_processes, + process_index=self.process_index, + split_batches=self.split_batches, + put_on_device=device_placement, + rng_types=self.rng_types.copy(), + dispatch_batches=self.dispatch_batches, + even_batches=self.even_batches, + slice_fn_for_dispatch=slice_fn_for_dispatch, + use_seedable_sampler=self.use_seedable_sampler, + data_seed=self.dataloader_config.data_seed, + non_blocking=self.non_blocking, + use_stateful_dataloader=self.use_stateful_dataloader, + torch_device_mesh=device_mesh, + ) + self._dataloaders.append(prepared_data_loader) + return prepared_data_loader + + def prepare_optimizer(self, optimizer: torch.optim.Optimizer, device_placement=None): + """ + Prepares a PyTorch Optimizer for training in any distributed setup. It is recommended to use + [`Accelerator.prepare`] instead. + + Args: + optimizer (`torch.optim.Optimizer`): + A vanilla PyTorch optimizer to prepare + device_placement (`bool`, *optional*): + Whether or not to place the optimizer on the proper device. Will default to `self.device_placement`. + + Example: + + ```python + >>> import torch + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> optimizer = torch.optim.Adam(...) + >>> optimizer = accelerator.prepare_optimizer(optimizer, device_placement=True) + ``` + """ + if is_lomo_available(): + # We need to import locally to avoid circular imports since lomo imports stuff from + # transformers & accelerate + from lomo_optim import AdaLomo, Lomo + + # Support multiple optimizers: https://github.com/huggingface/accelerate/pull/2695#discussion_r1589164607 + self.has_lomo_optimizer |= isinstance(optimizer, (Lomo, AdaLomo)) + + # Ensure we can't double wrap an optimizer due to `find_batch_size` + if getattr(optimizer, "_is_accelerate_prepared", False): + if optimizer not in self._optimizers: + self._optimizers.append(optimizer) + return optimizer + if device_placement is None: + device_placement = self.device_placement + # NOTE: Special case with MS-AMP we do *not* pass in the scaler explicitly to the `AcceleratedOptimizer`, + # Their optimizer handles it for us. + scaler = None if self.fp8_backend == FP8BackendType.MSAMP else self.scaler + optimizer = AcceleratedOptimizer(optimizer, device_placement=device_placement, scaler=scaler) + self._optimizers.append(optimizer) + return optimizer + + def prepare_scheduler(self, scheduler: LRScheduler): + """ + Prepares a PyTorch Scheduler for training in any distributed setup. It is recommended to use + [`Accelerator.prepare`] instead. + + Args: + scheduler (`torch.optim.lr_scheduler.LRScheduler`): + A vanilla PyTorch scheduler to prepare + + Example: + + ```python + >>> import torch + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> optimizer = torch.optim.Adam(...) + >>> scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, ...) + >>> scheduler = accelerator.prepare_scheduler(scheduler) + ``` + """ + # Ensure we can't double wrap a scheduler due to `find_batch_size` + if getattr(scheduler, "_is_accelerate_prepared", False): + if scheduler not in self._schedulers: + self._schedulers.append(scheduler) + return scheduler + # We try to find the optimizer associated with `scheduler`, the default is the full list. + optimizer = self._optimizers + for opt in self._optimizers: + if getattr(scheduler, "optimizer", None) == opt.optimizer: + optimizer = opt + break + scheduler = AcceleratedScheduler( + scheduler, + optimizer, + step_with_optimizer=self.step_scheduler_with_optimizer, + split_batches=self.split_batches, + ) + self._schedulers.append(scheduler) + return scheduler + + def backward(self, loss, **kwargs): + """ + Scales the gradients in accordance to the `GradientAccumulationPlugin` and calls the correct `backward()` based + on the configuration. + + Should be used in lieu of `loss.backward()`. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(gradient_accumulation_steps=2) + >>> outputs = model(inputs) + >>> loss = loss_fn(outputs, labels) + >>> accelerator.backward(loss) + ``` + """ + learning_rate = kwargs.get("learning_rate") + + if self.distributed_type != DistributedType.DEEPSPEED: + # deepspeed handles loss scaling by gradient_accumulation_steps in its `backward` + loss = loss / self.gradient_accumulation_steps + if self.distributed_type == DistributedType.DEEPSPEED: + self.deepspeed_engine_wrapped.backward(loss, sync_gradients=self.sync_gradients, **kwargs) + elif self.distributed_type == DistributedType.MEGATRON_LM: + return + elif self.scaler is not None: + self.scaler.scale(loss).backward(**kwargs) + elif learning_rate is not None and self.has_lomo_optimizer: + self.lomo_backward(loss, learning_rate) + else: + loss.backward(**kwargs) + + def set_trigger(self): + """ + Sets the internal trigger tensor to 1 on the current process. A latter check should follow using this which + will check across all processes. + + Note: + Does not require `wait_for_everyone()` + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> # Assume later in the training script + >>> # `should_do_breakpoint` is a custom function to monitor when to break, + >>> # e.g. when the loss is NaN + >>> if should_do_breakpoint(loss): + ... accelerator.set_trigger() + >>> # Assume later in the training script + >>> if accelerator.check_breakpoint(): + ... break + ``` + """ + self.flag_tensor = torch.tensor(1, device=self.device) + + def check_trigger(self): + """ + Checks if the internal trigger tensor has been set to 1 in any of the processes. If so, will return `True` and + reset the trigger tensor to 0. + + Note: + Does not require `wait_for_everyone()` + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> # Assume later in the training script + >>> # `should_do_breakpoint` is a custom function to monitor when to break, + >>> # e.g. when the loss is NaN + >>> if should_do_breakpoint(loss): + ... accelerator.set_trigger() + >>> # Assume later in the training script + >>> if accelerator.check_trigger(): + ... break + ``` + """ + # Now that we are outside `__init__`, we can initialize it if it is `None` on device + if self.flag_tensor is None: + self.flag_tensor = torch.tensor(0, device=self.device) + flag_tensor = self.reduce(self.flag_tensor) + if flag_tensor.item() >= 1: + self.flag_tensor = torch.tensor(0, device=self.device) + return True + return False + + def unscale_gradients(self, optimizer=None): + """ + Unscale the gradients in mixed precision training with AMP. This is a noop in all other settings. + + Likely should be called through [`Accelerator.clip_grad_norm_`] or [`Accelerator.clip_grad_value_`] + + Args: + optimizer (`torch.optim.Optimizer` or `list[torch.optim.Optimizer]`, *optional*): + The optimizer(s) for which to unscale gradients. If not set, will unscale gradients on all optimizers + that were passed to [`~Accelerator.prepare`]. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> model, optimizer = accelerator.prepare(model, optimizer) + >>> outputs = model(inputs) + >>> loss = loss_fn(outputs, labels) + >>> accelerator.backward(loss) + >>> accelerator.unscale_gradients(optimizer=optimizer) + ``` + """ + if self.native_amp and self.mixed_precision == "fp16": + if optimizer is None: + # TODO: this unscales all optimizers where we should only unscale the one where parameters are. + optimizer = self._optimizers + elif not isinstance(optimizer, (tuple, list)): + optimizer = [optimizer] + for opt in optimizer: + while isinstance(opt, AcceleratedOptimizer): + opt = opt.optimizer + self.scaler.unscale_(opt) + + def clip_grad_norm_(self, parameters, max_norm, norm_type=2): + """ + Should be used in place of `torch.nn.utils.clip_grad_norm_`. + + Returns: + `torch.Tensor`: Total norm of the parameter gradients (viewed as a single vector). + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(gradient_accumulation_steps=2) + >>> dataloader, model, optimizer, scheduler = accelerator.prepare(dataloader, model, optimizer, scheduler) + + >>> for input, target in dataloader: + ... optimizer.zero_grad() + ... output = model(input) + ... loss = loss_func(output, target) + ... accelerator.backward(loss) + ... if accelerator.sync_gradients: + ... accelerator.clip_grad_norm_(model.parameters(), max_grad_norm) + ... optimizer.step() + ``` + """ + if self.distributed_type == DistributedType.FSDP: + self.unscale_gradients() + parameters = [p for p in parameters] + for model in self._models: + if parameters == [p for p in model.parameters()]: + if not self.is_fsdp2: + return model.clip_grad_norm_(max_norm, norm_type) + else: + return torch.nn.utils.clip_grad_norm_( + parameters, max_norm, norm_type=norm_type + ) # viz: https://github.com/pytorch/torchtitan/blob/main/docs/fsdp.md + elif self.distributed_type == DistributedType.DEEPSPEED: + # DeepSpeed handles gradient clipping internally, but we can retrieve the gradient norm + if self.deepspeed_engine_wrapped is not None: + return self.deepspeed_engine_wrapped.get_global_grad_norm() + return None + elif self.distributed_type == DistributedType.XLA: + # Reduce gradients first for XLA + for acc_opt in self._optimizers: + if not acc_opt.gradient_state.is_xla_gradients_synced: + opt = acc_opt + while isinstance(opt, AcceleratedOptimizer): + opt = opt.optimizer + gradients = xm._fetch_gradients(opt) + # Use xm.all_reduce to perform an in-place all-reduce. Recursive all-reduce each tensor + # one by one in self.reduce is non-inplace. + xm.all_reduce("sum", gradients, scale=1.0 / self.num_processes) + # Set is_xla_gradients_synced to True to avoid all-reduce twice in the AcceleratedOptimizer step. + acc_opt.gradient_state.is_xla_gradients_synced = True + if os.environ.get("ACCELERATE_USE_FSDP", "false").lower() == "true": + self.unscale_gradients() + parameters = [p for p in parameters] + for model in self._models: + if parameters == [p for p in model.parameters()]: + return model.clip_grad_norm_(max_norm, norm_type) + self.unscale_gradients() + return torch.nn.utils.clip_grad_norm_(parameters, max_norm, norm_type=norm_type) + + def clip_grad_value_(self, parameters, clip_value): + """ + Should be used in place of `torch.nn.utils.clip_grad_value_`. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(gradient_accumulation_steps=2) + >>> dataloader, model, optimizer, scheduler = accelerator.prepare(dataloader, model, optimizer, scheduler) + + >>> for input, target in dataloader: + ... optimizer.zero_grad() + ... output = model(input) + ... loss = loss_func(output, target) + ... accelerator.backward(loss) + ... if accelerator.sync_gradients: + ... accelerator.clip_grad_value_(model.parameters(), clip_value) + ... optimizer.step() + ``` + """ + if self.distributed_type in [DistributedType.DEEPSPEED, DistributedType.FSDP]: + raise Exception("DeepSpeed and FSDP do not support `clip_grad_value_`. Use `clip_grad_norm_` instead.") + self.unscale_gradients() + torch.nn.utils.clip_grad_value_(parameters, clip_value) + + def gather(self, tensor): + """ + Gather the values in *tensor* across all processes and concatenate them on the first dimension. Useful to + regroup the predictions from all processes when doing evaluation. + + Note: + This gather happens in all processes. + + Args: + tensor (`torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`): + The tensors to gather across all processes. + + Returns: + `torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`: The gathered tensor(s). Note that the + first dimension of the result is *num_processes* multiplied by the first dimension of the input tensors. + + Example: + + ```python + >>> # Assuming four processes + >>> import torch + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> process_tensor = torch.tensor([accelerator.process_index], device=accelerator.device) + >>> gathered_tensor = accelerator.gather(process_tensor) + >>> gathered_tensor + tensor([0, 1, 2, 3]) + ``` + """ + return gather(tensor) + + def gather_for_metrics(self, input_data, use_gather_object=False): + """ + Gathers `input_data` and potentially drops duplicates in the last batch if on a distributed system. Should be + used for gathering the inputs and targets for metric calculation. + + Args: + input (`torch.Tensor`, `object`, a nested tuple/list/dictionary of `torch.Tensor`, or a nested tuple/list/dictionary of `object`): + The tensors or objects for calculating metrics across all processes + use_gather_object(`bool`): + Whether to forcibly use gather_object instead of gather (which is already done if all objects passed do + not contain tensors). This flag can be useful for gathering tensors with different sizes that we don't + want to pad and concatenate along the first dimension. Using it with GPU tensors is not well supported + and inefficient as it incurs GPU -> CPU transfer since tensors would be pickled. + + Example: + + ```python + >>> # Assuming two processes, with a batch size of 5 on a dataset with 9 samples + >>> import torch + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> dataloader = torch.utils.data.DataLoader(range(9), batch_size=5) + >>> dataloader = accelerator.prepare(dataloader) + >>> batch = next(iter(dataloader)) + >>> gathered_items = accelerator.gather_for_metrics(batch) + >>> len(gathered_items) + 9 + ``` + """ + + try: + recursively_apply(lambda x: x, input_data, error_on_other_type=True) + all_tensors = True + except TypeError: + all_tensors = False + + use_gather_object = use_gather_object or not all_tensors + + if use_gather_object: + data = gather_object(input_data) + else: + data = self.gather(input_data) + + try: + if self.gradient_state.end_of_dataloader: + # at the end of a dataloader, `gather_for_metrics` regresses to + # `gather` unless the dataset has a remainder so log. + if self.gradient_state.remainder == -1: + logger.info( + "The used dataset had no length, returning gathered tensors. You should drop the remainder yourself." + ) + return data + elif self.gradient_state.remainder > 0: + # Last batch needs to be truncated on distributed systems as it contains additional samples + def _adjust_samples(tensor): + return tensor[: self.gradient_state.remainder] + + if use_gather_object: + # gather_object put the objects in a list + return _adjust_samples(data) + else: + return recursively_apply(_adjust_samples, data) + else: # remainder is 0 + # no remainder even though at end of dataloader, so nothing to do. + return data + else: + # Not at the end of the dataloader, no need to adjust the tensors + return data + except Exception: + # Dataset had no length or raised an error + return data + + def reduce(self, tensor, reduction="sum", scale=1.0): + """ + Reduce the values in *tensor* across all processes based on *reduction*. + + Note: + All processes get the reduced value. + + Args: + tensor (`torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`): + The tensors to reduce across all processes. + reduction (`str`, *optional*, defaults to "sum"): + A reduction type, can be one of 'sum', 'mean', 'max', or 'none'. If 'none', will not perform any + operation. + scale (`float`, *optional*, defaults to 1.0): + A default scaling value to be applied after the reduce, only valid on XLA. + + Returns: + `torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`: + The reduced tensor(s). + + Example: + + ```python + >>> # Assuming two processes + >>> import torch + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> process_tensor = torch.arange(accelerator.num_processes) + 1 + (2 * accelerator.process_index) + >>> process_tensor = process_tensor.to(accelerator.device) + >>> reduced_tensor = accelerator.reduce(process_tensor, reduction="sum") + >>> reduced_tensor + tensor([4, 6]) + ``` + """ + return reduce(tensor, reduction, scale) + + def pad_across_processes(self, tensor, dim=0, pad_index=0, pad_first=False): + """ + Recursively pad the tensors in a nested list/tuple/dictionary of tensors from all devices to the same size so + they can safely be gathered. + + Args: + tensor (nested list/tuple/dictionary of `torch.Tensor`): + The data to gather. + dim (`int`, *optional*, defaults to 0): + The dimension on which to pad. + pad_index (`int`, *optional*, defaults to 0): + The value with which to pad. + pad_first (`bool`, *optional*, defaults to `False`): + Whether to pad at the beginning or the end. + + Returns: + `torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`: + The padded tensor(s). + + Example: + + ```python + >>> # Assuming two processes, with the first processes having a tensor of size 1 and the second of size 2 + >>> import torch + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> process_tensor = torch.arange(accelerator.process_index + 1).to(accelerator.device) + >>> padded_tensor = accelerator.pad_across_processes(process_tensor) + >>> padded_tensor.shape + torch.Size([2]) + ``` + """ + return pad_across_processes(tensor, dim=dim, pad_index=pad_index, pad_first=pad_first) + + def unwrap_model(self, model, keep_fp32_wrapper: bool = True, keep_torch_compile: bool = True): + """ + Unwraps the `model` from the additional layer possible added by [`~Accelerator.prepare`]. Useful before saving + the model. + + Args: + model (`torch.nn.Module`): + The model to unwrap. + keep_fp32_wrapper (`bool`, *optional*, defaults to `True`): + Whether to not remove the mixed precision hook if it was added. + keep_torch_compile (`bool`, *optional*, defaults to `True`): + Whether to not unwrap compiled model if compiled. + Returns: + `torch.nn.Module`: The unwrapped model. + + Example: + + ```python + >>> # Assuming two GPU processes + >>> from torch.nn.parallel import DistributedDataParallel + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> model = accelerator.prepare(MyModel()) + >>> print(model.__class__.__name__) + DistributedDataParallel + + >>> model = accelerator.unwrap_model(model) + >>> print(model.__class__.__name__) + MyModel + ``` + """ + return extract_model_from_parallel(model, keep_fp32_wrapper, keep_torch_compile) + + def wait_for_everyone(self): + """ + Will stop the execution of the current process until every other process has reached that point (so this does + nothing when the script is only run in one process). Useful to do before saving a model. + + Example: + + ```python + >>> # Assuming two GPU processes + >>> import time + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> if accelerator.is_main_process: + ... time.sleep(2) + >>> else: + ... print("I'm waiting for the main process to finish its sleep...") + >>> accelerator.wait_for_everyone() + >>> # Should print on every process at the same time + >>> print("Everyone is here") + ``` + """ + wait_for_everyone() + + @on_main_process + def init_trackers(self, project_name: str, config: dict | None = None, init_kwargs: dict | None = None): + """ + Initializes a run for all trackers stored in `self.log_with`, potentially with starting configurations + + Args: + project_name (`str`): + The name of the project. All trackers will save their data based on this + config (`dict`, *optional*): + Optional starting configuration to be logged. + init_kwargs (`dict`, *optional*): + A nested dictionary of kwargs to be passed to a specific tracker's `__init__` function. Should be + formatted like so: + ```python + {"wandb": {"tags": ["tag_a", "tag_b"]}} + ``` + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(log_with="tensorboard") + >>> accelerator.init_trackers( + ... project_name="my_project", + ... config={"learning_rate": 0.001, "batch_size": 32}, + ... init_kwargs={"tensorboard": {"flush_secs": 60}}, + ... ) + ``` + """ + if init_kwargs is None: + init_kwargs = {} + for tracker in self.log_with: + if issubclass(type(tracker), GeneralTracker): + # Custom trackers are already initialized + self.trackers.append(tracker) + else: + tracker_init = LOGGER_TYPE_TO_CLASS[str(tracker)] + if tracker_init.requires_logging_directory: + # We can skip this check since it was done in `__init__` + self.trackers.append( + tracker_init(project_name, self.logging_dir, **init_kwargs.get(str(tracker), {})) + ) + else: + self.trackers.append(tracker_init(project_name, **init_kwargs.get(str(tracker), {}))) + + for tracker in self.trackers: + tracker.start() + + if config is not None: + for tracker in self.trackers: + tracker.store_init_configuration(config) + + def get_tracker(self, name: str, unwrap: bool = False): + """ + Returns a `tracker` from `self.trackers` based on `name` on the main process only. + + Args: + name (`str`): + The name of a tracker, corresponding to the `.name` property. + unwrap (`bool`): + Whether to return the internal tracking mechanism or to return the wrapped tracker instead + (recommended). + + Returns: + `GeneralTracker`: The tracker corresponding to `name` if it exists. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(log_with="tensorboard") + >>> accelerator.init_trackers("my_project") + >>> tensorboard_tracker = accelerator.get_tracker("tensorboard") + ``` + """ + if len(self.trackers) > 0: + for tracker in self.trackers: + if tracker.name == name: + return tracker.tracker if unwrap else tracker + raise ValueError(f"{name} is not an available tracker stored inside the `Accelerator`.") + # Handle tracker only made on main process + return GeneralTracker(_blank=True) + + @on_main_process + def log(self, values: dict, step: int | None = None, log_kwargs: dict | None = None): + """ + Logs `values` to all stored trackers in `self.trackers` on the main process only. + + Args: + values (`dict`): + Values should be a dictionary-like object containing only types `int`, `float`, or `str`. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + log_kwargs (`dict`, *optional*): + A nested dictionary of kwargs to be passed to a specific tracker's `log` function. Should be formatted + like so: + ```python + {"wandb": {"tags": ["tag_a", "tag_b"]}} + ``` + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(log_with="tensorboard") + >>> accelerator.init_trackers("my_project") + >>> accelerator.log({"loss": 0.5, "accuracy": 0.9}) + ``` + """ + if log_kwargs is None: + log_kwargs = {} + for tracker in self.trackers: + tracker.log(values, step=step, **log_kwargs.get(tracker.name, {})) + + def end_training(self): + """ + Runs any special end training behaviors, such as stopping trackers on the main process only or destroying + process group. Should always be called at the end of your script if using experiment tracking. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(log_with="tensorboard") + >>> accelerator.init_trackers("my_project") + >>> # Do training + >>> accelerator.end_training() + ``` + """ + for tracker in self.trackers: + tracker.finish() + + self.state.destroy_process_group() + + def save(self, obj, f, safe_serialization=False): + """ + Save the object passed to disk once per machine. Use in place of `torch.save`. + + Args: + obj (`object`): The object to save. + f (`str` or `os.PathLike`): Where to save the content of `obj`. + safe_serialization (`bool`, *optional*, defaults to `False`): Whether to save `obj` using `safetensors` + + Note: + If `save_on_each_node` was passed in as a `ProjectConfiguration`, will save the object once per node, + rather than only once on the main node. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> arr = [0, 1, 2, 3] + >>> accelerator.save(arr, "array.pkl") + ``` + """ + save( + obj, + f, + save_on_each_node=self.project_configuration.save_on_each_node, + safe_serialization=safe_serialization, + ) + + def save_model( + self, + model: torch.nn.Module, + save_directory: Union[str, os.PathLike], + max_shard_size: Union[int, str] = "10GB", + safe_serialization: bool = True, + ): + """ + Save a model so that it can be re-loaded using load_checkpoint_in_model + + Arguments: + model: (`torch.nn.Module`): + Model to be saved. The model can be wrapped or unwrapped. + save_directory (`str` or `os.PathLike`): + Directory to which to save. Will be created if it doesn't exist. + max_shard_size (`int` or `str`, *optional*, defaults to `"10GB"`): + The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size + lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`). + + + + If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard + which will be bigger than `max_shard_size`. + + + + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> model = ... + >>> accelerator.save_model(model, save_directory) + ``` + """ + + if os.path.isfile(save_directory): + logger.error(f"Provided path ({save_directory}) should be a directory, not a file") + return + + # get the state_dict of the model + if any(has_offloaded_params(module) for module in model.modules()): + state_dict = get_state_dict_offloaded_model(model) + else: + if any(param.device == torch.device("meta") for param in model.parameters()): + raise RuntimeError("You can't save the model since some parameters are on the meta device.") + state_dict = self.get_state_dict(model) + + # Case: DeepSpeed zero3 gets gathered and `state_dict` is empty + if state_dict is None: + return + os.makedirs(save_directory, exist_ok=True) + + if safe_serialization: + state_dict = clean_state_dict_for_safetensors(state_dict) + weights_name = SAFE_WEIGHTS_NAME if safe_serialization else WEIGHTS_NAME + filename_pattern = SAFE_WEIGHTS_PATTERN_NAME if safe_serialization else WEIGHTS_PATTERN_NAME + + from huggingface_hub import split_torch_state_dict_into_shards + + state_dict_split = split_torch_state_dict_into_shards( + state_dict, filename_pattern=filename_pattern, max_shard_size=max_shard_size + ) + + # Clean the folder from a previous save + for filename in os.listdir(save_directory): + full_filename = os.path.join(save_directory, filename) + # If we have a shard file that is not going to be replaced, we delete it, but only from the main process + # in distributed settings to avoid race conditions. + weights_no_suffix = weights_name.replace(".bin", "") + + # make sure that file to be deleted matches format of sharded file, e.g. pytorch_model-00001-of-00005 + filename_no_suffix = filename.replace(".bin", "") + reg = re.compile(r"(.*?)-\d{5}-of-\d{5}") + + if ( + filename.startswith(weights_no_suffix) + and os.path.isfile(full_filename) + and filename not in state_dict_split.filename_to_tensors.keys() + and reg.fullmatch(filename_no_suffix) is not None + and PartialState().is_main_process + ): + os.remove(full_filename) + + # Save the model + for filename, tensors in state_dict_split.filename_to_tensors.items(): + shard = {tensor: state_dict[tensor] for tensor in tensors} + self.save(shard, os.path.join(save_directory, filename), safe_serialization=safe_serialization) + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = SAFE_WEIGHTS_INDEX_NAME if safe_serialization else WEIGHTS_INDEX_NAME + save_index_file = os.path.join(save_directory, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + logger.info( + f"The model is bigger than the maximum size per checkpoint ({max_shard_size}) and is going to be " + f"split in {len(state_dict_split.filename_to_tensors)} checkpoint shards. You can find where each parameters has been saved in the " + f"index located at {save_index_file}." + ) + else: + path_to_weights = os.path.join(save_directory, WEIGHTS_NAME) + logger.info(f"Model weights saved in {path_to_weights}") + + def register_save_state_pre_hook(self, hook: Callable[..., None]) -> hooks.RemovableHandle: + """ + Registers a pre hook to be run before `save_checkpoint` is called in [`Accelerator.save_state`]. + + Args: + hook (`Callable`): + A function to be called in [`Accelerator.save_state`] before `save_checkpoint`. + + The hook should have the following signature: + + `hook(models: list[torch.nn.Module], weights: list[dict[str, torch.Tensor]], input_dir: str) -> None` + + The `models` argument are the models as saved in the accelerator state under `accelerator._models`, `weights` + argument are the state dicts of the `models`, and the `input_dir` argument is the `input_dir` argument passed + to [`Accelerator.load_state`]. + + + + Should only be used in conjunction with [`Accelerator.register_load_state_pre_hook`]. Can be useful to save + configurations in addition to model weights. Can also be used to overwrite model saving with a customized + method. In this case, make sure to remove already loaded weights from the weights list. + + + + Returns: + `torch.utils.hooks.RemovableHandle`: a handle that can be used to remove the added hook by calling + `handle.remove()` + """ + handle = hooks.RemovableHandle(self._save_model_state_pre_hook) + self._save_model_state_pre_hook[handle.id] = hook + return handle + + def save_state(self, output_dir: str | None = None, safe_serialization: bool = True, **save_model_func_kwargs): + """ + Saves the current states of the model, optimizer, scaler, RNG generators, and registered objects to a folder. + + If a `ProjectConfiguration` was passed to the `Accelerator` object with `automatic_checkpoint_naming` enabled + then checkpoints will be saved to `self.project_dir/checkpoints`. If the number of current saves is greater + than `total_limit` then the oldest save is deleted. Each checkpoint is saved in separate folders named + `checkpoint_`. + + Otherwise they are just saved to `output_dir`. + + + + Should only be used when wanting to save a checkpoint during training and restoring the state in the same + environment. + + + + Args: + output_dir (`str` or `os.PathLike`): + The name of the folder to save all relevant weights and states. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + save_model_func_kwargs (`dict`, *optional*): + Additional keyword arguments for saving model which can be passed to the underlying save function, such + as optional arguments for DeepSpeed's `save_checkpoint` function. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> model, optimizer, lr_scheduler = ... + >>> model, optimizer, lr_scheduler = accelerator.prepare(model, optimizer, lr_scheduler) + >>> accelerator.save_state(output_dir="my_checkpoint") + ``` + """ + if self.project_configuration.automatic_checkpoint_naming: + output_dir = os.path.join(self.project_dir, "checkpoints") + os.makedirs(output_dir, exist_ok=True) + if self.project_configuration.automatic_checkpoint_naming: + folders = [os.path.join(output_dir, folder) for folder in os.listdir(output_dir)] + if ( + self.project_configuration.total_limit is not None + and (len(folders) + 1 > self.project_configuration.total_limit) + and self.is_main_process + ): + + def _inner(folder): + return list(map(int, re.findall(r"[\/]?([0-9]+)(?=[^\/]*$)", folder)))[0] + + folders.sort(key=_inner) + logger.warning( + f"Deleting {len(folders) + 1 - self.project_configuration.total_limit} checkpoints to make room for new checkpoint." + ) + for folder in folders[: len(folders) + 1 - self.project_configuration.total_limit]: + shutil.rmtree(folder) + output_dir = os.path.join(output_dir, f"checkpoint_{self.save_iteration}") + if os.path.exists(output_dir): + raise ValueError( + f"Checkpoint directory {output_dir} ({self.save_iteration}) already exists. Please manually override `self.save_iteration` with what iteration to start with." + ) + self.wait_for_everyone() + os.makedirs(output_dir, exist_ok=True) + logger.info(f"Saving current state to {output_dir}") + + if self.distributed_type == DistributedType.XLA: + # Finish running the previous step before checkpointing + xm.mark_step() + + # Save the models taking care of FSDP and DeepSpeed nuances + weights = [] + for i, model in enumerate(self._models): + if self.distributed_type == DistributedType.FSDP: + logger.info("Saving FSDP model") + save_fsdp_model(self.state.fsdp_plugin, self, model, output_dir, i) + logger.info(f"FSDP Model saved to output dir {output_dir}") + elif self.distributed_type == DistributedType.DEEPSPEED: + logger.info("Saving DeepSpeed Model and Optimizer") + ckpt_id = f"{MODEL_NAME}" if i == 0 else f"{MODEL_NAME}_{i}" + model.save_checkpoint(output_dir, ckpt_id, **save_model_func_kwargs) + logger.info(f"DeepSpeed Model and Optimizer saved to output dir {os.path.join(output_dir, ckpt_id)}") + elif self.distributed_type == DistributedType.MEGATRON_LM: + logger.info("Saving Megatron-LM Model, Optimizer and Scheduler") + model.save_checkpoint(output_dir) + logger.info(f"Megatron-LM Model , Optimizer and Scheduler saved to output dir {output_dir}") + else: + weights.append(self.get_state_dict(model, unwrap=False)) + + # Save the optimizers taking care of FSDP and DeepSpeed nuances + optimizers = [] + if self.distributed_type == DistributedType.FSDP: + for i, opt in enumerate(self._optimizers): + logger.info("Saving FSDP Optimizer") + save_fsdp_optimizer(self.state.fsdp_plugin, self, opt, self._models[i], output_dir, i) + logger.info(f"FSDP Optimizer saved to output dir {output_dir}") + elif self.distributed_type not in [DistributedType.DEEPSPEED, DistributedType.MEGATRON_LM]: + optimizers = self._optimizers + + # Save the lr schedulers taking care of DeepSpeed nuances + schedulers = [] + if self.distributed_type == DistributedType.DEEPSPEED: + for i, scheduler in enumerate(self._schedulers): + if isinstance(scheduler, DeepSpeedSchedulerWrapper): + continue + schedulers.append(scheduler) + elif self.distributed_type not in [DistributedType.MEGATRON_LM]: + schedulers = self._schedulers + + # Save the samplers of the dataloaders + dataloaders = self._dataloaders + + # Call model loading hooks that might have been registered with + # accelerator.register_model_state_hook + for hook in self._save_model_state_pre_hook.values(): + hook(self._models, weights, output_dir) + + save_location = save_accelerator_state( + output_dir, + weights, + optimizers, + schedulers, + dataloaders, + self.state.process_index, + self.step, + self.scaler, + save_on_each_node=self.project_configuration.save_on_each_node, + safe_serialization=safe_serialization, + ) + for i, obj in enumerate(self._custom_objects): + save_custom_state(obj, output_dir, i, save_on_each_node=self.project_configuration.save_on_each_node) + self.project_configuration.iteration += 1 + return save_location + + def register_load_state_pre_hook(self, hook: Callable[..., None]) -> hooks.RemovableHandle: + """ + Registers a pre hook to be run before [`load_checkpoint`] is called in [`Accelerator.load_state`]. + + Args: + hook (`Callable`): + A function to be called in [`Accelerator.load_state`] before `load_checkpoint`. + + The hook should have the following signature: + + `hook(models: list[torch.nn.Module], input_dir: str) -> None` + + The `models` argument are the models as saved in the accelerator state under `accelerator._models`, and the + `input_dir` argument is the `input_dir` argument passed to [`Accelerator.load_state`]. + + + + Should only be used in conjunction with [`Accelerator.register_save_state_pre_hook`]. Can be useful to load + configurations in addition to model weights. Can also be used to overwrite model loading with a customized + method. In this case, make sure to remove already loaded models from the models list. + + + + Returns: + `torch.utils.hooks.RemovableHandle`: a handle that can be used to remove the added hook by calling + `handle.remove()` + """ + handle = hooks.RemovableHandle(self._load_model_state_pre_hook) + self._load_model_state_pre_hook[handle.id] = hook + return handle + + def load_state(self, input_dir: str | None = None, load_kwargs: dict | None = None, **load_model_func_kwargs): + """ + Loads the current states of the model, optimizer, scaler, RNG generators, and registered objects. + + + + Should only be used in conjunction with [`Accelerator.save_state`]. If a file is not registered for + checkpointing, it will not be loaded if stored in the directory. + + + + Args: + input_dir (`str` or `os.PathLike`): + The name of the folder all relevant weights and states were saved in. Can be `None` if + `automatic_checkpoint_naming` is used, and will pick up from the latest checkpoint. + load_kwargs (`dict`, *optional*): + Additional keyword arguments for the underlying `load` function, such as optional arguments for + state_dict and optimizer on. + load_model_func_kwargs (`dict`, *optional*): + Additional keyword arguments for loading model which can be passed to the underlying load function, + such as optional arguments for DeepSpeed's `load_checkpoint` function or a `map_location` to load the + model and optimizer on. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> model, optimizer, lr_scheduler = ... + >>> model, optimizer, lr_scheduler = accelerator.prepare(model, optimizer, lr_scheduler) + >>> accelerator.load_state("my_checkpoint") + ``` + """ + if input_dir is not None: + # Check if folder exists + input_dir = os.path.expanduser(input_dir) + if not os.path.isdir(input_dir): + raise ValueError(f"Tried to find {input_dir} but folder does not exist") + elif self.project_configuration.automatic_checkpoint_naming: + # Pick up from automatic checkpoint naming + input_dir = os.path.join(self.project_dir, "checkpoints") + folders = [os.path.join(input_dir, folder) for folder in os.listdir(input_dir)] + + def _inner(folder): + return list(map(int, re.findall(r"[\/]?([0-9]+)(?=[^\/]*$)", folder)))[0] + + folders.sort(key=_inner) + input_dir = folders[-1] + else: + raise ValueError("No input_dir provided and automatic checkpoint naming is disabled.") + logger.info(f"Loading states from {input_dir}") + + # Load the models taking care of FSDP and DeepSpeed nuances + models = [] + for i, model in enumerate(self._models): + if self.distributed_type == DistributedType.FSDP: + logger.info("Loading FSDP model") + load_fsdp_model(self.state.fsdp_plugin, self, model, input_dir, i) + logger.info(f"FSDP Model loaded from input dir {input_dir}") + elif self.distributed_type == DistributedType.DEEPSPEED: + logger.info("Loading DeepSpeed Model and Optimizer") + ckpt_id = f"{MODEL_NAME}" if i == 0 else f"{MODEL_NAME}_{i}" + model.load_checkpoint(input_dir, ckpt_id, **load_model_func_kwargs) + logger.info(f"DeepSpeed Model and Optimizer loaded from input dir {os.path.join(input_dir, ckpt_id)}") + elif self.distributed_type == DistributedType.MEGATRON_LM: + logger.info("Loading Megatron-LM Model, Optimizer and Scheduler") + model.load_checkpoint(input_dir) + logger.info(f"Megatron-LM Model , Optimizer and Scheduler loaded from input dir {input_dir}") + else: + models.append(model) + + # We need to load the scaler state before the optimizer for FSDP2 + # (`torch.distributed.checkpoint.set_optimizer_state_dict`) which we use to set the state of the optimizer calls `optimizer.step` on + # a dummy tensor, but since the scaler is not initialized, it will raise an error (the scaler exists but its `_scale` is None) + scaler = None + if self.scaler is not None and self.is_fsdp2: + input_scaler_file = os.path.join(input_dir, SCALER_NAME) + scaler_state = torch.load(input_scaler_file) + self.scaler.load_state_dict(scaler_state) + # We also need to call the `_lazy_init_scale_growth_tracker` to initialize the scaler, as it would else be called + # on the first call to scale + self.scaler._lazy_init_scale_growth_tracker(self.scaler._device) + logger.info("GradScaler state loaded successfully") + else: + scaler = self.scaler + + # Load the optimizers taking care of FSDP and DeepSpeed nuances + optimizers = [] + if self.distributed_type == DistributedType.FSDP: + for i, opt in enumerate(self._optimizers): + logger.info("Loading FSDP Optimizer") + load_fsdp_optimizer(self.state.fsdp_plugin, self, opt, self._models[i], input_dir, i) + logger.info(f"FSDP Optimizer loaded from input dir {input_dir}") + elif self.distributed_type not in [DistributedType.DEEPSPEED, DistributedType.MEGATRON_LM]: + optimizers = self._optimizers + + # Load the lr schedulers taking care of DeepSpeed nuances + schedulers = [] + if self.distributed_type == DistributedType.DEEPSPEED: + for i, scheduler in enumerate(self._schedulers): + if isinstance(scheduler, DeepSpeedSchedulerWrapper): + continue + schedulers.append(scheduler) + elif self.distributed_type not in [DistributedType.MEGATRON_LM]: + schedulers = self._schedulers + + dataloaders = self._dataloaders + + # Call model loading hooks that might have been registered with + # accelerator.register_model_state_hook + for hook in self._load_model_state_pre_hook.values(): + hook(models, input_dir) + + map_location = load_model_func_kwargs.pop("map_location", None) + if map_location is None: + if self.num_processes > 1 and self.multi_device and self.distributed_type != DistributedType.MULTI_XPU: + map_location = "on_device" + else: + map_location = "cpu" + + override_attributes = load_accelerator_state( + input_dir, + models, + optimizers, + schedulers, + dataloaders, + self.state.process_index, + scaler, + map_location, + load_kwargs, + **load_model_func_kwargs, + ) + if "step" in override_attributes: + self.step = override_attributes["step"] + custom_checkpoints = [ + f for f in os.listdir(input_dir) if re.search(r"^custom_checkpoint_\d+\.pkl$", f) is not None + ] + if len(custom_checkpoints) != len(self._custom_objects): + err = ( + f"Number of custom checkpoints in folder {input_dir} does not match the number of registered objects:" + ) + err += f"\n\tFound checkpoints: {len(custom_checkpoints)}" + err += f"\n\tRegistered objects: {len(self._custom_objects)}\n" + err += "Please make sure to only load checkpoints from folders that were created with the same set of registered objects," + err += "or avoid using `custom_checkpoint` in the filename for files in that same directory and load them in manually." + raise RuntimeError(err) + else: + logger.info(f"Loading in {len(custom_checkpoints)} custom states") + for index, obj in enumerate(self._custom_objects): + load_custom_state(obj, input_dir, index) + + def free_memory(self, *objects): + """ + Will release all references to the internal objects stored and call the garbage collector. You should call this + method between two trainings with different models/optimizers. Also will reset `Accelerator.step` to 0. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> model, optimizer, scheduler = ... + >>> model, optimizer, scheduler = accelerator.prepare(model, optimizer, scheduler) + >>> model, optimizer, scheduler = accelerator.free_memory(model, optimizer, scheduler) + ``` + """ + # Deepspeed needs a bit more prep that should be done first + if hasattr(self, "deepspeed_engine_wrapped"): + if self.deepspeed_engine_wrapped is not None: + self.deepspeed_engine_wrapped.engine.destroy() + self.deepspeed_engine_wrapped = None + objects = release_memory(*objects) + self._schedulers = [] + self._optimizers = [] + self._models = [] + self._dataloaders = [] + self.step = 0 + return objects + + def clear(self, *objects): + """ + Alias for [`Accelerate.free_memory`], releases all references to the internal objects stored and call the + garbage collector. You should call this method between two trainings with different models/optimizers. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> model, optimizer, scheduler = ... + >>> model, optimizer, scheduler = accelerator.prepare(model, optimizer, scheduler) + >>> model, optimizer, scheduler = accelerator.clear(model, optimizer, scheduler) + ``` + """ + return self.free_memory(*objects) + + def _get_named_parameters(self, *args, drop_refs=False): + named_parameters = {} + accessor_mapping = {} + for obj in args: + if isinstance(obj, torch.nn.Module): + obj = extract_model_from_parallel(obj) + if not drop_refs: + named_parameters.update({n: p for n, p in obj.named_parameters()}) + continue + + # we need this bit as `WeightWithDynamic...` returns 0 when `data_ptr()` is called, + # the underlying pointer is actually hidden in `_tensor` attribute + if self.fp8_backend == FP8BackendType.AO: + from torchao.float8.fsdp_utils import WeightWithDynamicFloat8CastTensor + + accessor_mapping[WeightWithDynamicFloat8CastTensor] = "_tensor" + _torch_distributed_available = torch.distributed.is_available() + _is_dtensor_available = _torch_distributed_available and is_torch_version( + ">=", DTENSOR_PYTORCH_VERSION + ) + # we know we're in FSDP2 so DTensor is available + if _is_dtensor_available: + from torch.distributed.tensor import DTensor + + accessor_mapping[DTensor] = "_local_tensor" + + named_parameters.update( + { + n: getattr(p, accessor_mapping[type(p)]).data_ptr() + if type(p) in accessor_mapping + else p.data_ptr() + for n, p in obj.named_parameters() + } + ) + return named_parameters + + def _get_devices(self, *args): + model_device = None + optimizer_device = None + for obj in args: + # Loop through model parameters and stop at the first once we have its device. + if isinstance(obj, torch.nn.Module): + for param in obj.parameters(): + model_device = param.device + break + # Loop through optimizer parameters groups and stop at the first once we have its device. + if isinstance(obj, torch.optim.Optimizer): + for param_group in obj.param_groups: + if len(param_group["params"]) > 0: + optimizer_device = param_group["params"][0].device + break + return (model_device, optimizer_device) + + def get_state_dict(self, model, unwrap=True): + """ + Returns the state dictionary of a model sent through [`Accelerator.prepare`] potentially without full + precision. + + Args: + model (`torch.nn.Module`): + A PyTorch model sent through [`Accelerator.prepare`] + unwrap (`bool`, *optional*, defaults to `True`): + Whether to return the original underlying state_dict of `model` or to return the wrapped state_dict + + Returns: + `dict`: The state dictionary of the model potentially without full precision. + + Example: + + ```python + >>> import torch + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> net = torch.nn.Linear(2, 2) + >>> net = accelerator.prepare(net) + >>> state_dict = accelerator.get_state_dict(net) + ``` + """ + + if self.distributed_type == DistributedType.DEEPSPEED: + zero3_sharding = self.deepspeed_config["zero_optimization"]["stage"] == 3 + tp_sharding = self.deepspeed_config.get("tensor_parallel", {}).get("autotp_size", 0) > 1 + if zero3_sharding or tp_sharding: + if model.zero_gather_16bit_weights_on_model_save(): + ver_min_required = "0.16.4" + if tp_sharding and not compare_versions("deepspeed", ">=", ver_min_required): + raise ImportError( + f"Deepspeed TP requires deepspeed>={ver_min_required}. Please update DeepSpeed via `pip install deepspeed -U`." + ) + state_dict = ( + model._consolidated_16bit_state_dict() + if tp_sharding + else model._zero3_consolidated_16bit_state_dict() + ) + else: + raise ValueError( + "Cannot get 16bit model weights because `stage3_gather_16bit_weights_on_model_save` in DeepSpeed config is False. " + "To save the model weights in 16bit, set `stage3_gather_16bit_weights_on_model_save` to True in DeepSpeed config file or " + "set `zero3_save_16bit_model` to True when using `accelerate config`. " + "To save the full checkpoint, run `model.save_checkpoint(save_dir)` and use `zero_to_fp32.py` to recover weights." + ) + else: + from deepspeed.checkpoint.utils import clone_tensors_for_torch_save + + state_dict = clone_tensors_for_torch_save(self.unwrap_model(model).state_dict()) + elif self.is_fsdp2: + from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict + + options = StateDictOptions(full_state_dict=True, broadcast_from_rank0=True, cpu_offload=True) + state_dict = get_model_state_dict(model, options=options) + elif self.distributed_type == DistributedType.FSDP: + from torch.distributed.fsdp import FullStateDictConfig, StateDictType + from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + + full_state_dict_config = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + with FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT, full_state_dict_config): + state_dict = model.state_dict() + else: + if unwrap: + model = self.unwrap_model(model) + state_dict = model.state_dict() + + return state_dict + + def register_for_checkpointing(self, *objects): + """ + Makes note of `objects` and will save or load them in during `save_state` or `load_state`. + + These should be utilized when the state is being loaded or saved in the same script. It is not designed to be + used in different scripts. + + + + Every `object` must have a `load_state_dict` and `state_dict` function to be stored. + + + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> # Assume `CustomObject` has a `state_dict` and `load_state_dict` function. + >>> obj = CustomObject() + >>> accelerator.register_for_checkpointing(obj) + >>> accelerator.save_state("checkpoint.pt") + ``` + """ + invalid_objects = [] + for obj in objects: + if not hasattr(obj, "state_dict") or not hasattr(obj, "load_state_dict"): + invalid_objects.append(obj) + if len(invalid_objects) > 0: + err = "All `objects` must include a `state_dict` and `load_state_dict` function to be stored. The following inputs are invalid:" + for index, obj in enumerate(invalid_objects): + err += f"\n\t- Item at index {index}, `{get_pretty_name(obj)}`" + raise ValueError(err) + self._custom_objects.extend(objects) + + @contextmanager + def maybe_context_parallel( + self, + buffers: list[torch.Tensor] | None = None, + buffer_seq_dims: list[int] | None = None, + no_restore_buffers: set[torch.Tensor] | None = None, + ): + """ + A context manager that enables context parallel training. + + Args: + buffers (`list[torch.Tensor]`, `optional`): + Buffers, which are going to be sharded along the sequence dimension. Common examples are inputs, labels + or positional embedding buffers. This context manager will modify these buffers in-place, and after + exiting the context, the buffers will be restored to their original state. To avoid unnecessary + restores, you can use `no_restore_buffers` to specify which buffers don't need to be restored. + buffer_seq_dims (`list[int]`, `optional`): + Sequence dimensions of `buffers`. + no_restore_buffers (`set[torch.Tensor]`, `optional`): + This set must be a subset of `buffers`. Specifies which buffers from `buffers` argument won't be + restored after the context exits. These buffers will be then kept in sharded state. + + + + `context_parallel` is currently supported with FSDP2 and requires `parallelism_config.cp_size` > + 1. If either of these conditions are not met, this context manager will have no effect, though to enable fewer + code changes it will not raise an Exception. + + + + + + This context manager has to be recreated with each training step, as shown in the example below. + + + + Example: + + ```python + >>> for batch in dataloader: + ... with accelerator.maybe_context_parallel( + ... buffers=[batch["input_ids"], batch["attention_mask"]], + ... buffer_seq_dims=[1, 1], + ... no_restore_buffers={batch["input_ids"]}, + ... ): + ... outputs = model(batch) + ... ... + ``` + """ + # We don't need to check FSDP2 as parallelism_config does that for us + # Invariant: in this branch self._cp_context is set, as it was set by `self._prepare_cp` + if ( + self.parallelism_config + and self.parallelism_config.cp_backend == "torch" + and self.parallelism_config.cp_enabled + ): + with self._cp_context( + buffers=buffers, buffer_seq_dims=buffer_seq_dims, no_restore_buffers=no_restore_buffers + ): + yield + else: + logger.warning_once( + "Context parallel training is not enabled. This context manager will have no effect. " + "To enable it, set `parallelism_config.cp_size` > 1 in the `Accelerator` constructor." + ) + yield + + @contextmanager + def autocast(self, autocast_handler: AutocastKwargs = None): + """ + Will apply automatic mixed-precision inside the block inside this context manager, if it is enabled. Nothing + different will happen otherwise. + + A different `autocast_handler` can be passed in to override the one set in the `Accelerator` object. This is + useful in blocks under `autocast` where you want to revert to fp32. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator(mixed_precision="fp16") + >>> with accelerator.autocast(): + ... train() + ``` + """ + if autocast_handler is None: + autocast_handler = self.autocast_handler + autocast_context = get_mixed_precision_context_manager(self.native_amp, autocast_handler) + with autocast_context: + yield + + @contextmanager + def profile(self, profile_handler: ProfileKwargs | None = None): + """ + Will profile the code inside the context manager. The profile will be saved to a Chrome Trace file if + `profile_handler.output_trace_dir` is set. + + A different `profile_handler` can be passed in to override the one set in the `Accelerator` object. + + Args: + profile_handler (`ProfileKwargs`, *optional*): + The profile handler to use for this context manager. If not passed, will use the one set in the + `Accelerator` object. + + Example: + + ```python + # Profile with default settings + from accelerate import Accelerator + from accelerate.utils import ProfileKwargs + + accelerator = Accelerator() + with accelerator.profile() as prof: + train() + accelerator.print(prof.key_averages().table()) + + + # Profile with the custom handler + def custom_handler(prof): + print(prof.key_averages().table(sort_by="self_cpu_time_total", row_limit=10)) + + + kwargs = ProfileKwargs(schedule_option=dict(wait=1, warmup=1, active=1), on_trace_ready=custom_handler) + accelerator = Accelerator(kwarg_handler=[kwargs]) + with accelerator.profile() as prof: + for _ in range(10): + train_iteration() + prof.step() + + + # Profile and export to Chrome Trace + kwargs = ProfileKwargs(output_trace_dir="output_trace") + accelerator = Accelerator(kwarg_handler=[kwargs]) + with accelerator.profile(): + train() + ``` + """ + profile_handler = profile_handler or self.profile_handler or ProfileKwargs() + + with profile_handler.build() as profiler: + yield profiler + + if profile_handler.output_trace_dir is None: + return + + os.makedirs(profile_handler.output_trace_dir, exist_ok=True) + profiler.export_chrome_trace( + os.path.join(profile_handler.output_trace_dir, PROFILE_PATTERN_NAME.format(suffix=self.process_index)) + ) + self.wait_for_everyone() + + @property + def optimizer_step_was_skipped(self): + """ + Whether or not the optimizer update was skipped (because of gradient overflow in mixed precision), in which + case the learning rate should not be changed. + """ + for optimizer in self._optimizers: + if optimizer.step_was_skipped: + return True + return False + + def skip_first_batches(self, dataloader, num_batches: int = 0): + """ + Creates a new `torch.utils.data.DataLoader` that will efficiently skip the first `num_batches`. + + Args: + dataloader (`torch.utils.data.DataLoader`): The data loader in which to skip batches. + num_batches (`int`, *optional*, defaults to 0): The number of batches to skip + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> dataloader, model, optimizer, scheduler = accelerator.prepare(dataloader, model, optimizer, scheduler) + >>> skipped_dataloader = accelerator.skip_first_batches(dataloader, num_batches=2) + >>> # for the first epoch only + >>> for input, target in skipped_dataloader: + ... optimizer.zero_grad() + ... output = model(input) + ... loss = loss_func(output, target) + ... accelerator.backward(loss) + ... optimizer.step() + + >>> # subsequent epochs + >>> for input, target in dataloader: + ... optimizer.zero_grad() + ... ... + ``` + """ + return skip_first_batches(dataloader, num_batches=num_batches) + + def __deepcopy__(self, memo): + logger.info("Deep copying the `Accelerator` object, note that this will point to the same original object.") + return self + + def verify_device_map(self, model: torch.nn.Module) -> bool: + """ + Verifies that `model` has not been prepared with big model inference with a device-map resembling `auto`. + """ + # Checks if any of the child modules has the attribute `hf_device_map` and this map has more than one entry. + for m in model.modules(): + if hasattr(m, "hf_device_map") and len(m.hf_device_map) > 1: + return True + + return False + + def lomo_backward(self, loss: torch.Tensor, learning_rate: float) -> None: + """ + Runs backward pass on LOMO optimizers. + """ + if is_lomo_available(): + # We need to import locally to avoid circular imports since lomo imports stuff from + # transformers & accelerate + from lomo_optim import AdaLomo, Lomo + + if learning_rate is None: + raise ValueError("A learning rate must be passed in order to call backward pass with LOMO optimizers.") + + _backward_called = False + + for optimizer in self._optimizers: + if isinstance(optimizer.optimizer, (Lomo, AdaLomo)): + optimizer.optimizer.fused_backward(loss, learning_rate) + _backward_called = True + + if not _backward_called: + raise ValueError( + "Backward pass not properly called on LOMO optimizers. Are you sure you passed a LOMO optimizer in accelerator.prepare()?" + ) + + @property + def fp8_backend(self) -> FP8BackendType: + "Returns the configured backend for training in FP8" + if self.has_fp8_handler: + if self.fp8_recipe_handler is not None: + return FP8BackendType(self.fp8_recipe_handler.backend) + elif self.ao_recipe_handler is not None: + return FP8BackendType.AO + elif self.te_recipe_handler is not None: + return FP8BackendType.TE + elif self.msamp_recipe_handler is not None: + return FP8BackendType.MSAMP + elif self.state.deepspeed_plugin is not None and self.state.deepspeed_plugin.enable_msamp: + return FP8BackendType.MSAMP + + return FP8BackendType(parse_choice_from_env("ACCELERATE_FP8_BACKEND", "NO")) diff --git a/kbidu_env/Lib/site-packages/accelerate/big_modeling.py b/kbidu_env/Lib/site-packages/accelerate/big_modeling.py new file mode 100644 index 0000000000000000000000000000000000000000..ac629efe8baed921d4c835f2f7c797dac56bb1ba --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/big_modeling.py @@ -0,0 +1,799 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import re +from contextlib import contextmanager +from functools import wraps +from typing import Optional, Union + +import torch +import torch.nn as nn + +from .hooks import ( + AlignDevicesHook, + CpuOffload, + LayerwiseCastingHook, + UserCpuOffloadHook, + add_hook_to_module, + attach_align_device_hook, + attach_align_device_hook_on_blocks, +) +from .utils import ( + OffloadedWeightsLoader, + check_cuda_p2p_ib_support, + check_device_map, + extract_submodules_state_dict, + find_tied_parameters, + get_balanced_memory, + infer_auto_device_map, + is_bnb_available, + is_mlu_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_xpu_available, + load_checkpoint_in_model, + offload_state_dict, + parse_flag_from_env, + retie_parameters, +) +from .utils.constants import SUPPORTED_PYTORCH_LAYERS_FOR_UPCASTING +from .utils.other import recursive_getattr + + +logger = logging.getLogger(__name__) + + +@contextmanager +def init_empty_weights(include_buffers: Optional[bool] = None): + """ + A context manager under which models are initialized with all parameters on the meta device, therefore creating an + empty model. Useful when just initializing the model would blow the available RAM. + + Args: + include_buffers (`bool`, *optional*): + Whether or not to also put all buffers on the meta device while initializing. + + Example: + + ```python + import torch.nn as nn + from accelerate import init_empty_weights + + # Initialize a model with 100 billions parameters in no time and without using any RAM. + with init_empty_weights(): + tst = nn.Sequential(*[nn.Linear(10000, 10000) for _ in range(1000)]) + ``` + + + + Any model created under this context manager has no weights. As such you can't do something like + `model.to(some_device)` with it. To load weights inside your empty model, see [`load_checkpoint_and_dispatch`]. + Make sure to overwrite the default device_map param for [`load_checkpoint_and_dispatch`], otherwise dispatch is not + called. + + + """ + if include_buffers is None: + include_buffers = parse_flag_from_env("ACCELERATE_INIT_INCLUDE_BUFFERS", False) + with init_on_device(torch.device("meta"), include_buffers=include_buffers) as f: + yield f + + +@contextmanager +def init_on_device(device: torch.device, include_buffers: Optional[bool] = None): + """ + A context manager under which models are initialized with all parameters on the specified device. + + Args: + device (`torch.device`): + Device to initialize all parameters on. + include_buffers (`bool`, *optional*): + Whether or not to also put all buffers on the meta device while initializing. + + Example: + + ```python + import torch.nn as nn + from accelerate import init_on_device + + # init model on specified device(e.g., "cuda", "xpu" and so on) + with init_on_device(device=torch.device("cuda")): + tst = nn.Linear(100, 100) # on specified device + ``` + """ + if include_buffers is None: + include_buffers = parse_flag_from_env("ACCELERATE_INIT_INCLUDE_BUFFERS", False) + + if include_buffers: + with device: + yield + return + + old_register_parameter = nn.Module.register_parameter + if include_buffers: + old_register_buffer = nn.Module.register_buffer + + def register_empty_parameter(module, name, param): + old_register_parameter(module, name, param) + if param is not None: + param_cls = type(module._parameters[name]) + kwargs = module._parameters[name].__dict__ + kwargs["requires_grad"] = param.requires_grad + # Pop non-constructor attributes before creating the parameter, then restore them after + _is_hf_initialized = kwargs.pop("_is_hf_initialized", None) + module._parameters[name] = param_cls(module._parameters[name].to(device), **kwargs) + if _is_hf_initialized is not None: + module._parameters[name]._is_hf_initialized = _is_hf_initialized + + def register_empty_buffer(module, name, buffer, persistent=True): + old_register_buffer(module, name, buffer, persistent=persistent) + if buffer is not None: + module._buffers[name] = module._buffers[name].to(device) + + # Patch tensor creation + if include_buffers: + tensor_constructors_to_patch = { + torch_function_name: getattr(torch, torch_function_name) + for torch_function_name in ["empty", "zeros", "ones", "full"] + } + else: + tensor_constructors_to_patch = {} + + def patch_tensor_constructor(fn): + def wrapper(*args, **kwargs): + kwargs["device"] = device + return fn(*args, **kwargs) + + return wrapper + + try: + nn.Module.register_parameter = register_empty_parameter + if include_buffers: + nn.Module.register_buffer = register_empty_buffer + for torch_function_name in tensor_constructors_to_patch.keys(): + setattr(torch, torch_function_name, patch_tensor_constructor(getattr(torch, torch_function_name))) + yield + finally: + nn.Module.register_parameter = old_register_parameter + if include_buffers: + nn.Module.register_buffer = old_register_buffer + for torch_function_name, old_torch_function in tensor_constructors_to_patch.items(): + setattr(torch, torch_function_name, old_torch_function) + + +def cpu_offload( + model: nn.Module, + execution_device: Optional[torch.device] = None, + offload_buffers: bool = False, + state_dict: Optional[dict[str, torch.Tensor]] = None, + preload_module_classes: Optional[list[str]] = None, +): + """ + Activates full CPU offload for a model. As a result, all parameters of the model will be offloaded and only one + copy of the state dict of the model will be kept. During the forward pass, parameters will be extracted from that + state dict and put on the execution device passed as they are needed, then offloaded again. + + Args: + model (`torch.nn.Module`): + The model to offload. + execution_device (`torch.device`, *optional*): + The device on which the forward pass of the model will be executed (should be a GPU). Will default to the + model first parameter device. + offload_buffers (`bool`, *optional*, defaults to `False`): + Whether or not to offload the buffers with the model parameters. + state_dict (`Dict[str, torch.Tensor]`, *optional*): + The state dict of the model that will be kept on CPU. + preload_module_classes (`List[str]`, *optional*): + A list of classes whose instances should load all their weights (even in the submodules) at the beginning + of the forward. This should only be used for classes that have submodules which are registered but not + called directly during the forward, for instance if a `dense` linear layer is registered, but at forward, + `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly. + """ + if execution_device is None: + execution_device = next(iter(model.parameters())).device + if state_dict is None: + state_dict = {n: p.to("cpu") for n, p in model.state_dict().items()} + + add_hook_to_module(model, AlignDevicesHook(io_same_device=True), append=True) + attach_align_device_hook( + model, + execution_device=execution_device, + offload=True, + offload_buffers=offload_buffers, + weights_map=state_dict, + preload_module_classes=preload_module_classes, + ) + + return model + + +def cpu_offload_with_hook( + model: torch.nn.Module, + execution_device: Optional[Union[int, str, torch.device]] = None, + prev_module_hook: Optional[UserCpuOffloadHook] = None, +): + """ + Offloads a model on the CPU and puts it back to an execution device when executed. The difference with + [`cpu_offload`] is that the model stays on the execution device after the forward and is only offloaded again when + the `offload` method of the returned `hook` is called. Useful for pipelines running a model in a loop. + + Args: + model (`torch.nn.Module`): + The model to offload. + execution_device(`str`, `int` or `torch.device`, *optional*): + The device on which the model should be executed. Will default to the MPS device if it's available, then + device 0 if there is an accelerator device, and finally to the CPU. + prev_module_hook (`UserCpuOffloadHook`, *optional*): + The hook sent back by this function for a previous model in the pipeline you are running. If passed, its + offload method will be called just before the forward of the model to which this hook is attached. + + Example: + + ```py + model_1, hook_1 = cpu_offload_with_hook(model_1, device) + model_2, hook_2 = cpu_offload_with_hook(model_2, device, prev_module_hook=hook_1) + model_3, hook_3 = cpu_offload_with_hook(model_3, device, prev_module_hook=hook_2) + + hid_1 = model_1(input) + for i in range(50): + # model1 is offloaded on the CPU at the first iteration, model 2 stays on the GPU for this whole loop. + hid_2 = model_2(hid_1) + # model2 is offloaded to the CPU just before this forward. + hid_3 = model_3(hid_3) + + # For model3, you need to manually call the hook offload method. + hook_3.offload() + ``` + """ + hook = CpuOffload(execution_device=execution_device, prev_module_hook=prev_module_hook) + add_hook_to_module(model, hook, append=True) + user_hook = UserCpuOffloadHook(model, hook) + return model, user_hook + + +def disk_offload( + model: nn.Module, + offload_dir: Union[str, os.PathLike], + execution_device: Optional[torch.device] = None, + offload_buffers: bool = False, + preload_module_classes: Optional[list[str]] = None, +): + """ + Activates full disk offload for a model. As a result, all parameters of the model will be offloaded as + memory-mapped array in a given folder. During the forward pass, parameters will be accessed from that folder and + put on the execution device passed as they are needed, then offloaded again. + + Args: + model (`torch.nn.Module`): The model to offload. + offload_dir (`str` or `os.PathLike`): + The folder in which to offload the model weights (or where the model weights are already offloaded). + execution_device (`torch.device`, *optional*): + The device on which the forward pass of the model will be executed (should be a GPU). Will default to the + model's first parameter device. + offload_buffers (`bool`, *optional*, defaults to `False`): + Whether or not to offload the buffers with the model parameters. + preload_module_classes (`List[str]`, *optional*): + A list of classes whose instances should load all their weights (even in the submodules) at the beginning + of the forward. This should only be used for classes that have submodules which are registered but not + called directly during the forward, for instance if a `dense` linear layer is registered, but at forward, + `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly. + """ + if not os.path.isdir(offload_dir) or not os.path.isfile(os.path.join(offload_dir, "index.json")): + offload_state_dict(offload_dir, model.state_dict()) + if execution_device is None: + execution_device = next(iter(model.parameters())).device + weights_map = OffloadedWeightsLoader(save_folder=offload_dir) + + add_hook_to_module(model, AlignDevicesHook(io_same_device=True), append=True) + attach_align_device_hook( + model, + execution_device=execution_device, + offload=True, + offload_buffers=offload_buffers, + weights_map=weights_map, + preload_module_classes=preload_module_classes, + ) + + return model + + +def dispatch_model( + model: nn.Module, + device_map: dict[str, Union[str, int, torch.device]], + main_device: Optional[torch.device] = None, + state_dict: Optional[dict[str, torch.Tensor]] = None, + offload_dir: Optional[Union[str, os.PathLike]] = None, + offload_index: Optional[dict[str, str]] = None, + offload_buffers: bool = False, + skip_keys: Optional[Union[str, list[str]]] = None, + preload_module_classes: Optional[list[str]] = None, + force_hooks: bool = False, +): + """ + Dispatches a model according to a given device map. Layers of the model might be spread across GPUs, offloaded on + the CPU or even the disk. + + Args: + model (`torch.nn.Module`): + The model to dispatch. + device_map (`Dict[str, Union[str, int, torch.device]]`): + A dictionary mapping module names in the models `state_dict` to the device they should go to. Note that + `"disk"` is accepted even if it's not a proper value for `torch.device`. + main_device (`str`, `int` or `torch.device`, *optional*): + The main execution device. Will default to the first device in the `device_map` different from `"cpu"` or + `"disk"`. + state_dict (`Dict[str, torch.Tensor]`, *optional*): + The state dict of the part of the model that will be kept on CPU. + offload_dir (`str` or `os.PathLike`): + The folder in which to offload the model weights (or where the model weights are already offloaded). + offload_index (`Dict`, *optional*): + A dictionary from weight name to their information (`dtype`/ `shape` or safetensors filename). Will default + to the index saved in `save_folder`. + offload_buffers (`bool`, *optional*, defaults to `False`): + Whether or not to offload the buffers with the model parameters. + skip_keys (`str` or `List[str]`, *optional*): + A list of keys to ignore when moving inputs or outputs between devices. + preload_module_classes (`List[str]`, *optional*): + A list of classes whose instances should load all their weights (even in the submodules) at the beginning + of the forward. This should only be used for classes that have submodules which are registered but not + called directly during the forward, for instance if a `dense` linear layer is registered, but at forward, + `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly. + force_hooks (`bool`, *optional*, defaults to `False`): + Whether or not to force device hooks to be attached to the model even if all layers are dispatched to a + single device. + """ + # Error early if the device map is incomplete. + check_device_map(model, device_map) + + # We need to force hook for quantized model that can't be moved with to() + if getattr(model, "quantization_method", "bitsandbytes") == "bitsandbytes": + # since bnb 0.43.2, we can move 4-bit model + if (getattr(model, "is_loaded_in_8bit", False) and not is_bnb_available(min_version="0.48.0")) or ( + getattr(model, "is_loaded_in_4bit", False) and not is_bnb_available(min_version="0.43.2") + ): + force_hooks = True + + # We attach hooks if the device_map has at least 2 different devices or if + # force_hooks is set to `True`. Otherwise, the model in already loaded + # in the unique device and the user can decide where to dispatch the model. + # If the model is quantized, we always force-dispatch the model + if (len(set(device_map.values())) > 1) or force_hooks: + if main_device is None: + if set(device_map.values()) == {"cpu"} or set(device_map.values()) == {"cpu", "disk"}: + main_device = "cpu" + else: + main_device = [d for d in device_map.values() if d not in ["cpu", "disk"]][0] + + if main_device != "cpu": + cpu_modules = [name for name, device in device_map.items() if device == "cpu"] + if state_dict is None and len(cpu_modules) > 0: + state_dict = extract_submodules_state_dict(model.state_dict(), cpu_modules) + + disk_modules = [name for name, device in device_map.items() if device == "disk"] + if offload_dir is None and offload_index is None and len(disk_modules) > 0: + raise ValueError( + "We need an `offload_dir` to dispatch this model according to this `device_map`, the following submodules " + f"need to be offloaded: {', '.join(disk_modules)}." + ) + if ( + len(disk_modules) > 0 + and offload_index is None + and (not os.path.isdir(offload_dir) or not os.path.isfile(os.path.join(offload_dir, "index.json"))) + ): + disk_state_dict = extract_submodules_state_dict(model.state_dict(), disk_modules) + offload_state_dict(offload_dir, disk_state_dict) + + execution_device = { + name: main_device if device in ["cpu", "disk"] else device for name, device in device_map.items() + } + execution_device[""] = main_device + offloaded_devices = ["disk"] if main_device == "cpu" or main_device == "mps" else ["cpu", "disk"] + offload = {name: device in offloaded_devices for name, device in device_map.items()} + save_folder = offload_dir if len(disk_modules) > 0 else None + if state_dict is not None or save_folder is not None or offload_index is not None: + device = main_device if offload_index is not None else None + weights_map = OffloadedWeightsLoader( + state_dict=state_dict, save_folder=save_folder, index=offload_index, device=device + ) + else: + weights_map = None + + # When dispatching the model's parameters to the devices specified in device_map, we want to avoid allocating memory several times for the + # tied parameters. The dictionary tied_params_map keeps track of the already allocated data for a given tied parameter (represented by its + # original pointer) on each devices. + tied_params = find_tied_parameters(model) + + tied_params_map = {} + for group in tied_params: + for param_name in group: + # data_ptr() is enough here, as `find_tied_parameters` finds tied params simply by comparing `param1 is param2`, so we don't need + # to care about views of tensors through storage_offset. + data_ptr = recursive_getattr(model, param_name).data_ptr() + tied_params_map[data_ptr] = {} + + # Note: To handle the disk offloading case, we can not simply use weights_map[param_name].data_ptr() as the reference pointer, + # as we have no guarantee that safetensors' `file.get_tensor()` will always give the same pointer. + + attach_align_device_hook_on_blocks( + model, + execution_device=execution_device, + offload=offload, + offload_buffers=offload_buffers, + weights_map=weights_map, + skip_keys=skip_keys, + preload_module_classes=preload_module_classes, + tied_params_map=tied_params_map, + ) + + # warn if there is any params on the meta device + offloaded_devices_str = " and ".join( + [device for device in set(device_map.values()) if device in ("cpu", "disk")] + ) + if len(offloaded_devices_str) > 0: + logger.warning( + f"Some parameters are on the meta device because they were offloaded to the {offloaded_devices_str}." + ) + + # Attaching the hook may break tied weights, so we retie them + retie_parameters(model, tied_params) + + # add warning on `to` method + def add_warning(fn, model): + @wraps(fn) + def wrapper(*args, **kwargs): + warning_msg = "You shouldn't move a model that is dispatched using accelerate hooks." + if str(fn.__name__) == "to": + to_device = torch._C._nn._parse_to(*args, **kwargs)[0] + if to_device is not None: + logger.warning(warning_msg) + else: + logger.warning(warning_msg) + for param in model.parameters(): + if param.device == torch.device("meta"): + raise RuntimeError("You can't move a model that has some modules offloaded to cpu or disk.") + return fn(*args, **kwargs) + + return wrapper + + # Make sure to update _accelerate_added_attributes in hooks.py if you add any hook + model.to = add_warning(model.to, model) + if is_npu_available(): + model.npu = add_warning(model.npu, model) + elif is_mlu_available(): + model.mlu = add_warning(model.mlu, model) + elif is_sdaa_available(): + model.sdaa = add_warning(model.sdaa, model) + elif is_musa_available(): + model.musa = add_warning(model.musa, model) + elif is_xpu_available(): + model.xpu = add_warning(model.xpu, model) + elif is_neuron_available(): + model.neuron = add_warning(model.neuron, model) + else: + model.cuda = add_warning(model.cuda, model) + + # Check if we are using multi-gpus with RTX 4000 series + use_multi_gpu = len([device for device in set(device_map.values()) if device not in ("cpu", "disk")]) > 1 + if use_multi_gpu and not check_cuda_p2p_ib_support(): + logger.warning( + "We've detected an older driver with an RTX 4000 series GPU. These drivers have issues with P2P. " + "This can affect the multi-gpu inference when using accelerate device_map." + "Please make sure to update your driver to the latest version which resolves this." + ) + else: + device = list(device_map.values())[0] + # `torch.Tensor.to()` is not supported by `torch_npu` (see this [issue](https://github.com/Ascend/pytorch/issues/16)). + if is_npu_available() and isinstance(device, int): + device = f"npu:{device}" + elif is_mlu_available() and isinstance(device, int): + device = f"mlu:{device}" + elif is_sdaa_available() and isinstance(device, int): + device = f"sdaa:{device}" + elif is_musa_available() and isinstance(device, int): + device = f"musa:{device}" + elif is_neuron_available() and isinstance(device, int): + device = f"neuron:{device}" + if device != "disk": + model.to(device) + else: + raise ValueError( + "You are trying to offload the whole model to the disk. Please use the `disk_offload` function instead." + ) + # Convert OrderedDict back to dict for easier usage + model.hf_device_map = dict(device_map) + return model + + +def load_checkpoint_and_dispatch( + model: nn.Module, + checkpoint: Union[str, os.PathLike], + device_map: Optional[Union[str, dict[str, Union[int, str, torch.device]]]] = None, + max_memory: Optional[dict[Union[int, str], Union[int, str]]] = None, + no_split_module_classes: Optional[list[str]] = None, + offload_folder: Optional[Union[str, os.PathLike]] = None, + offload_buffers: bool = False, + dtype: Optional[Union[str, torch.dtype]] = None, + offload_state_dict: Optional[bool] = None, + skip_keys: Optional[Union[str, list[str]]] = None, + preload_module_classes: Optional[list[str]] = None, + force_hooks: bool = False, + strict: bool = False, + full_state_dict: bool = True, + broadcast_from_rank0: bool = False, +): + """ + Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are + loaded and adds the various hooks that will make this model run properly (even if split across devices). + + Args: + model (`torch.nn.Module`): The model in which we want to load a checkpoint. + checkpoint (`str` or `os.PathLike`): + The folder checkpoint to load. It can be: + - a path to a file containing a whole model state dict + - a path to a `.json` file containing the index to a sharded checkpoint + - a path to a folder containing a unique `.index.json` file and the shards of a checkpoint. + device_map (`Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer + name, once a given module name is inside, every submodule of it will be sent to the same device. + + To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For more + information about each option see [here](../concept_guides/big_model_inference#designing-a-device-map). + Defaults to None, which means [`dispatch_model`] will not be called. + max_memory (`Dict`, *optional*): + A dictionary device identifier to maximum memory. Will default to the maximum memory available for each GPU + and the available CPU RAM if unset. + no_split_module_classes (`List[str]`, *optional*): + A list of layer class names that should never be split across device (for instance any layer that has a + residual connection). + offload_folder (`str` or `os.PathLike`, *optional*): + If the `device_map` contains any value `"disk"`, the folder where we will offload weights. + offload_buffers (`bool`, *optional*, defaults to `False`): + In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as + well as the parameters. + dtype (`str` or `torch.dtype`, *optional*): + If provided, the weights will be converted to that type when loaded. + offload_state_dict (`bool`, *optional*): + If `True`, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if + the weight of the CPU state dict + the biggest shard does not fit. Will default to `True` if the device map + picked contains `"disk"` values. + skip_keys (`str` or `List[str]`, *optional*): + A list of keys to ignore when moving inputs or outputs between devices. + preload_module_classes (`List[str]`, *optional*): + A list of classes whose instances should load all their weights (even in the submodules) at the beginning + of the forward. This should only be used for classes that have submodules which are registered but not + called directly during the forward, for instance if a `dense` linear layer is registered, but at forward, + `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly. + force_hooks (`bool`, *optional*, defaults to `False`): + Whether or not to force device hooks to be attached to the model even if all layers are dispatched to a + single device. + strict (`bool`, *optional*, defaults to `False`): + Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model's + state_dict. + full_state_dict (`bool`, *optional*, defaults to `True`): if this is set to `True`, all the tensors in the + loaded state_dict will be gathered. No ShardedTensor and DTensor will be in the loaded state_dict. + broadcast_from_rank0 (`False`, *optional*, defaults to `False`): when the option is `True`, a distributed + `ProcessGroup` must be initialized. rank0 should receive a full state_dict and will broadcast the tensors + in the state_dict one by one to other ranks. Other ranks will receive the tensors and shard (if applicable) + according to the local shards in the model. + + Example: + + ```python + >>> from accelerate import init_empty_weights, load_checkpoint_and_dispatch + >>> from huggingface_hub import hf_hub_download + >>> from transformers import AutoConfig, AutoModelForCausalLM + + >>> # Download the Weights + >>> checkpoint = "EleutherAI/gpt-j-6B" + >>> weights_location = hf_hub_download(checkpoint, "pytorch_model.bin") + + >>> # Create a model and initialize it with empty weights + >>> config = AutoConfig.from_pretrained(checkpoint) + >>> with init_empty_weights(): + ... model = AutoModelForCausalLM.from_config(config) + + >>> # Load the checkpoint and dispatch it to the right devices + >>> model = load_checkpoint_and_dispatch( + ... model, weights_location, device_map="auto", no_split_module_classes=["GPTJBlock"] + ... ) + ``` + """ + if isinstance(device_map, str) and device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]: + raise ValueError( + "If passing a string for `device_map`, please choose 'auto', 'balanced', 'balanced_low_0' or 'sequential'." + ) + if isinstance(device_map, str): + if device_map != "sequential": + max_memory = get_balanced_memory( + model, + max_memory=max_memory, + no_split_module_classes=no_split_module_classes, + dtype=dtype, + low_zero=(device_map == "balanced_low_0"), + ) + device_map = infer_auto_device_map( + model, + max_memory=max_memory, + no_split_module_classes=no_split_module_classes, + dtype=dtype, + offload_buffers=offload_buffers, + ) + if offload_state_dict is None and device_map is not None and "disk" in device_map.values(): + offload_state_dict = True + load_checkpoint_in_model( + model, + checkpoint, + device_map=device_map, + offload_folder=offload_folder, + dtype=dtype, + offload_state_dict=offload_state_dict, + offload_buffers=offload_buffers, + strict=strict, + full_state_dict=full_state_dict, + broadcast_from_rank0=broadcast_from_rank0, + ) + if device_map is None: + return model + return dispatch_model( + model, + device_map=device_map, + offload_dir=offload_folder, + offload_buffers=offload_buffers, + skip_keys=skip_keys, + preload_module_classes=preload_module_classes, + force_hooks=force_hooks, + ) + + +def attach_layerwise_casting_hooks( + module: torch.nn.Module, + storage_dtype: torch.dtype, + compute_dtype: torch.dtype, + skip_modules_pattern: Optional[Union[str, tuple[str, ...]]] = None, + skip_modules_classes: Optional[tuple[type[torch.nn.Module], ...]] = None, + non_blocking: bool = False, +) -> None: + r""" + Applies layerwise casting to a given module. The module expected here is a PyTorch `nn.Module`. This is helpful for + reducing memory requirements when one doesn't want to fully quantize a model. Model params can be kept in say, + `torch.float8_e4m3fn` and upcasted to a higher precision like `torch.bfloat16` during forward pass and downcasted + back to `torch.float8_e4m3fn` to realize memory savings. + + Args: + module (`torch.nn.Module`): + The module whose leaf modules will be cast to a high precision dtype for computation, and to a low + precision dtype for storage. + storage_dtype (`torch.dtype`): + The dtype to cast the module to before/after the forward pass for storage. + compute_dtype (`torch.dtype`): + The dtype to cast the module to during the forward pass for computation. + skip_modules_pattern (`tuple[str, ...]`, defaults to `None`): + A list of patterns to match the names of the modules to skip during the layerwise casting process. If set + to `None` alongside `skip_modules_classes` being `None`, the layerwise casting is applied directly to the + module instead of its internal submodules. + skip_modules_classes (`tuple[type[torch.nn.Module], ...]`, defaults to `None`): + A list of module classes to skip during the layerwise casting process. + non_blocking (`bool`, defaults to `False`): + If `True`, the weight casting operations are non-blocking. + + Example: + + ```python + >>> from accelerate.hooks import attach_layerwise_casting_hooks + >>> from transformers import AutoModelForCausalLM + >>> import torch + + >>> # Model + >>> checkpoint = "EleutherAI/gpt-j-6B" + >>> model = AutoModelForCausalLM.from_pretrained(checkpoint) + + >>> # Attach hooks and perform inference + >>> attach_layerwise_casting_hooks(model, storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16) + >>> with torch.no_grad(): + ... model(...) + ``` + + Users can also pass modules they want to avoid from getting downcasted. + + ```py + >>> attach_layerwise_casting_hooks( + ... model, storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16, skip_modules_pattern=["norm"] + ... ) + ``` + """ + _attach_layerwise_casting_hooks( + module, storage_dtype, compute_dtype, skip_modules_pattern, skip_modules_classes, non_blocking + ) + + +def _attach_layerwise_casting_hooks( + module: torch.nn.Module, + storage_dtype: torch.dtype, + compute_dtype: torch.dtype, + skip_modules_pattern: Optional[Union[str, tuple[str, ...]]] = None, + skip_modules_classes: Optional[tuple[type[torch.nn.Module], ...]] = None, + non_blocking: bool = False, + _prefix: str = "", +): + should_skip = (skip_modules_classes is not None and isinstance(module, skip_modules_classes)) or ( + skip_modules_pattern is not None and any(re.search(pattern, _prefix) for pattern in skip_modules_pattern) + ) + if should_skip: + logger.debug(f'Skipping layerwise casting for layer "{_prefix}"') + return + + if isinstance(module, SUPPORTED_PYTORCH_LAYERS_FOR_UPCASTING): + logger.debug(f'Applying layerwise casting to layer "{_prefix}"') + add_hook_to_module( + module, + LayerwiseCastingHook(storage_dtype=storage_dtype, compute_dtype=compute_dtype, non_blocking=non_blocking), + append=True, + ) + return + + for name, submodule in module.named_children(): + layer_name = f"{_prefix}.{name}" if _prefix else name + _attach_layerwise_casting_hooks( + submodule, + storage_dtype, + compute_dtype, + skip_modules_pattern, + skip_modules_classes, + non_blocking, + _prefix=layer_name, + ) + + +def _attach_context_parallel_hooks( + model: nn.Module, +): + """ + Monkeypatch huggingface's `transformers` model to fix attention mask issues when using context parallelism. + + This function attaches forward_pre_hooks to each self_attn module of the model, where each hook checks the + args/kwargs, if they contain an attention mask, if it does, it will remove this mask, check if it is a causal mask, + if yes, will add a kwarg `is_causal=True`, otherwise will raise an error. This is because context parallelism does + not support attention masks. This function modifies the model in place. + + Args: + model (`nn.Module`): + The model to attach the hooks to. + + """ + + def _self_attn_pre_forward_hook(_module, module_args, module_kwargs): + if "attention_mask" in module_kwargs: + module_kwargs["attention_mask"] = None + module_kwargs["is_causal"] = True + + return module_args, module_kwargs + + for name, module in model.named_modules(): + # We hope (assume) that if user uses their own model (without this structure which transformers uses), they read the docs saying they can't pass in attention masks + # Then these cases can happen: + # 1) some modules end with a `self-attn` module, in which case we attach the hook, but the + # there's no attention mask kwarg -> hook is a no-op + # 2) some modules end with a `self-attn` module, in which case we attach the hook, and the + # attention mask kwarg is passed -> hook will remove the attention mask and add + # `is_causal=True` kwarg, which either crashes the training or fixes it + # (training would crash anyway as attention mask isn't supported) + # 3) no modules end with a `self-attn` module, in which case we don't attach the hook, this is + # a no-op as well + if name.endswith("self_attn"): + # we want the hook to be executed first, to avoid any other hooks doing work on the attention mask + module.register_forward_pre_hook(_self_attn_pre_forward_hook, with_kwargs=True, prepend=True) diff --git a/kbidu_env/Lib/site-packages/accelerate/checkpointing.py b/kbidu_env/Lib/site-packages/accelerate/checkpointing.py new file mode 100644 index 0000000000000000000000000000000000000000..ef869ebea54a1504cbb1d772af95a55c25009b67 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/checkpointing.py @@ -0,0 +1,340 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +from pathlib import Path +from typing import Optional + +import numpy as np +import torch +from safetensors.torch import load_model + +from .utils import ( + MODEL_NAME, + OPTIMIZER_NAME, + RNG_STATE_NAME, + SAFE_MODEL_NAME, + SAFE_WEIGHTS_NAME, + SAMPLER_NAME, + SCALER_NAME, + SCHEDULER_NAME, + WEIGHTS_NAME, + get_pretty_name, + is_cuda_available, + is_hpu_available, + is_mlu_available, + is_musa_available, + is_neuron_available, + is_sdaa_available, + is_torch_version, + is_torch_xla_available, + is_xpu_available, + load, + save, +) + + +if is_torch_version(">=", "2.4.0"): + from torch.amp import GradScaler +else: + from torch.cuda.amp import GradScaler + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + +from .logging import get_logger +from .state import PartialState + + +logger = get_logger(__name__) + + +def save_accelerator_state( + output_dir: str, + model_states: list[dict], + optimizers: list, + schedulers: list, + dataloaders: list, + process_index: int, + step: int, + scaler: Optional[GradScaler] = None, + save_on_each_node: bool = False, + safe_serialization: bool = True, +): + """ + Saves the current states of the models, optimizers, scaler, and RNG generators to a given directory. + + + + If `safe_serialization` is `True`, models will be saved with `safetensors` while the rest are saved using native + `pickle`. + + + + Args: + output_dir (`str` or `os.PathLike`): + The name of the folder to save all relevant weights and states. + model_states (`List[torch.nn.Module]`): + A list of model states + optimizers (`List[torch.optim.Optimizer]`): + A list of optimizer instances + schedulers (`List[torch.optim.lr_scheduler._LRScheduler]`): + A list of learning rate schedulers + dataloaders (`List[torch.utils.data.DataLoader]`): + A list of dataloader instances to save their sampler states + process_index (`int`): + The current process index in the Accelerator state + step (`int`): + The current step in the internal step tracker + scaler (`torch.amp.GradScaler`, *optional*): + An optional gradient scaler instance to save; + save_on_each_node (`bool`, *optional*): + Whether to save on every node, or only the main node. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + """ + output_dir = Path(output_dir) + # Model states + for i, state in enumerate(model_states): + weights_name = WEIGHTS_NAME if not safe_serialization else SAFE_WEIGHTS_NAME + if i > 0: + weights_name = weights_name.replace(".", f"_{i}.") + output_model_file = output_dir.joinpath(weights_name) + save(state, output_model_file, save_on_each_node=save_on_each_node, safe_serialization=safe_serialization) + logger.info(f"Model weights saved in {output_model_file}") + # Optimizer states + for i, opt in enumerate(optimizers): + state = opt.state_dict() + optimizer_name = f"{OPTIMIZER_NAME}.bin" if i == 0 else f"{OPTIMIZER_NAME}_{i}.bin" + output_optimizer_file = output_dir.joinpath(optimizer_name) + save(state, output_optimizer_file, save_on_each_node=save_on_each_node, safe_serialization=False) + logger.info(f"Optimizer state saved in {output_optimizer_file}") + # Scheduler states + for i, scheduler in enumerate(schedulers): + state = scheduler.state_dict() + scheduler_name = f"{SCHEDULER_NAME}.bin" if i == 0 else f"{SCHEDULER_NAME}_{i}.bin" + output_scheduler_file = output_dir.joinpath(scheduler_name) + save(state, output_scheduler_file, save_on_each_node=save_on_each_node, safe_serialization=False) + logger.info(f"Scheduler state saved in {output_scheduler_file}") + # DataLoader states + for i, dataloader in enumerate(dataloaders): + sampler_name = f"{SAMPLER_NAME}.bin" if i == 0 else f"{SAMPLER_NAME}_{i}.bin" + output_sampler_file = output_dir.joinpath(sampler_name) + # Only save if we have our custom sampler + from .data_loader import IterableDatasetShard, SeedableRandomSampler + + if isinstance(dataloader.dataset, IterableDatasetShard): + sampler = dataloader.get_sampler() + if isinstance(sampler, SeedableRandomSampler): + save(sampler, output_sampler_file, save_on_each_node=save_on_each_node, safe_serialization=False) + if getattr(dataloader, "use_stateful_dataloader", False): + dataloader_state_dict_name = "dl_state_dict.bin" if i == 0 else f"dl_state_dict_{i}.bin" + output_dataloader_state_dict_file = output_dir.joinpath(dataloader_state_dict_name) + state_dict = dataloader.state_dict() + torch.save(state_dict, output_dataloader_state_dict_file) + logger.info(f"Sampler state for dataloader {i} saved in {output_sampler_file}") + + # GradScaler state + if scaler is not None: + state = scaler.state_dict() + output_scaler_file = output_dir.joinpath(SCALER_NAME) + torch.save(state, output_scaler_file) + logger.info(f"Gradient scaler state saved in {output_scaler_file}") + # Random number generator states + states = {} + states_name = f"{RNG_STATE_NAME}_{process_index}.pkl" + states["step"] = step + states["random_state"] = random.getstate() + states["numpy_random_seed"] = np.random.get_state() + states["torch_manual_seed"] = torch.get_rng_state() + if is_xpu_available(): + states["torch_xpu_manual_seed"] = torch.xpu.get_rng_state_all() + if is_mlu_available(): + states["torch_mlu_manual_seed"] = torch.mlu.get_rng_state_all() + elif is_sdaa_available(): + states["torch_sdaa_manual_seed"] = torch.sdaa.get_rng_state_all() + elif is_musa_available(): + states["torch_musa_manual_seed"] = torch.musa.get_rng_state_all() + if is_hpu_available(): + states["torch_hpu_manual_seed"] = torch.hpu.get_rng_state_all() + if is_neuron_available(): + states["torch_neuron_manual_seed"] = torch.neuron.get_rng_state_all() + if is_cuda_available(): + states["torch_cuda_manual_seed"] = torch.cuda.get_rng_state_all() + if is_torch_xla_available(): + states["xm_seed"] = xm.get_rng_state() + output_states_file = output_dir.joinpath(states_name) + torch.save(states, output_states_file) + logger.info(f"Random states saved in {output_states_file}") + return output_dir + + +def load_accelerator_state( + input_dir, + models, + optimizers, + schedulers, + dataloaders, + process_index, + scaler=None, + map_location=None, + load_kwargs=None, + **load_model_func_kwargs, +): + """ + Loads states of the models, optimizers, scaler, and RNG generators from a given directory. + + Args: + input_dir (`str` or `os.PathLike`): + The name of the folder to load all relevant weights and states. + models (`List[torch.nn.Module]`): + A list of model instances + optimizers (`List[torch.optim.Optimizer]`): + A list of optimizer instances + schedulers (`List[torch.optim.lr_scheduler._LRScheduler]`): + A list of learning rate schedulers + dataloaders (`List[torch.utils.data.DataLoader]`): + A list of dataloader instances used in your program + process_index (`int`): + The current process index in the Accelerator state + scaler (`torch.amp.GradScaler`, *optional*): + An optional *GradScaler* instance to load + map_location (`str`, *optional*): + What device to load the optimizer state onto. Should be one of either "cpu" or "on_device". + load_kwargs (`dict`, *optional*): + Additional arguments that can be passed to the `load` function. + load_model_func_kwargs (`dict`, *optional*): + Additional arguments that can be passed to the model's `load_state_dict` method. + + Returns: + `dict`: Contains the `Accelerator` attributes to override while loading the state. + """ + # stores the `Accelerator` attributes to override + override_attributes = dict() + if map_location not in [None, "cpu", "on_device"]: + raise TypeError( + "Unsupported optimizer map location passed, please choose one of `None`, `'cpu'`, or `'on_device'`" + ) + if map_location is None: + map_location = "cpu" + elif map_location == "on_device": + map_location = PartialState().device + + if load_kwargs is None: + load_kwargs = {} + + input_dir = Path(input_dir) + # Model states + for i, model in enumerate(models): + ending = f"_{i}" if i > 0 else "" + input_model_file = input_dir.joinpath(f"{SAFE_MODEL_NAME}{ending}.safetensors") + if input_model_file.exists(): + load_model(model, input_model_file, device=str(map_location), **load_model_func_kwargs) + else: + # Load with torch + input_model_file = input_dir.joinpath(f"{MODEL_NAME}{ending}.bin") + state_dict = load(input_model_file, map_location=map_location) + model.load_state_dict(state_dict, **load_model_func_kwargs) + logger.info("All model weights loaded successfully") + + # Optimizer states + for i, opt in enumerate(optimizers): + optimizer_name = f"{OPTIMIZER_NAME}.bin" if i == 0 else f"{OPTIMIZER_NAME}_{i}.bin" + input_optimizer_file = input_dir.joinpath(optimizer_name) + optimizer_state = load(input_optimizer_file, map_location=map_location, **load_kwargs) + optimizers[i].load_state_dict(optimizer_state) + logger.info("All optimizer states loaded successfully") + + # Scheduler states + for i, scheduler in enumerate(schedulers): + scheduler_name = f"{SCHEDULER_NAME}.bin" if i == 0 else f"{SCHEDULER_NAME}_{i}.bin" + input_scheduler_file = input_dir.joinpath(scheduler_name) + scheduler_state = load(input_scheduler_file, **load_kwargs) + scheduler.load_state_dict(scheduler_state) + logger.info("All scheduler states loaded successfully") + + for i, dataloader in enumerate(dataloaders): + sampler_name = f"{SAMPLER_NAME}.bin" if i == 0 else f"{SAMPLER_NAME}_{i}.bin" + input_sampler_file = input_dir.joinpath(sampler_name) + # Only load if we have our custom sampler + from .data_loader import IterableDatasetShard, SeedableRandomSampler + + if isinstance(dataloader.dataset, IterableDatasetShard): + sampler = dataloader.get_sampler() + if isinstance(sampler, SeedableRandomSampler): + sampler = dataloader.set_sampler(load(input_sampler_file)) + if getattr(dataloader, "use_stateful_dataloader", False): + dataloader_state_dict_name = "dl_state_dict.bin" if i == 0 else f"dl_state_dict_{i}.bin" + input_dataloader_state_dict_file = input_dir.joinpath(dataloader_state_dict_name) + if input_dataloader_state_dict_file.exists(): + state_dict = load(input_dataloader_state_dict_file, **load_kwargs) + dataloader.load_state_dict(state_dict) + logger.info("All dataloader sampler states loaded successfully") + + # GradScaler state + if scaler is not None: + input_scaler_file = input_dir.joinpath(SCALER_NAME) + scaler_state = load(input_scaler_file) + scaler.load_state_dict(scaler_state) + logger.info("GradScaler state loaded successfully") + + # Random states + try: + states = load(input_dir.joinpath(f"{RNG_STATE_NAME}_{process_index}.pkl")) + if "step" in states: + override_attributes["step"] = states["step"] + random.setstate(states["random_state"]) + np.random.set_state(states["numpy_random_seed"]) + torch.set_rng_state(states["torch_manual_seed"]) + if is_xpu_available(): + torch.xpu.set_rng_state_all(states["torch_xpu_manual_seed"]) + if is_mlu_available(): + torch.mlu.set_rng_state_all(states["torch_mlu_manual_seed"]) + elif is_sdaa_available(): + torch.sdaa.set_rng_state_all(states["torch_sdaa_manual_seed"]) + elif is_musa_available(): + torch.musa.set_rng_state_all(states["torch_musa_manual_seed"]) + elif is_hpu_available(): + torch.hpu.set_rng_state_all(states["torch_hpu_manual_seed"]) + elif is_neuron_available(): + torch.neuron.set_rng_state_all(states["torch_neuron_manual_seed"]) + else: + torch.cuda.set_rng_state_all(states["torch_cuda_manual_seed"]) + if is_torch_xla_available(): + xm.set_rng_state(states["xm_seed"]) + logger.info("All random states loaded successfully") + except Exception: + logger.info("Could not load random states") + + return override_attributes + + +def save_custom_state(obj, path, index: int = 0, save_on_each_node: bool = False): + """ + Saves the state of `obj` to `{path}/custom_checkpoint_{index}.pkl` + """ + # Should this be the right way to get a qual_name type value from `obj`? + save_location = Path(path) / f"custom_checkpoint_{index}.pkl" + logger.info(f"Saving the state of {get_pretty_name(obj)} to {save_location}") + save(obj.state_dict(), save_location, save_on_each_node=save_on_each_node) + + +def load_custom_state(obj, path, index: int = 0): + """ + Loads the state of `obj` at `{path}/custom_checkpoint_{index}.pkl`. Will always set `weights_only=False` when + loading the state. + """ + load_location = f"{path}/custom_checkpoint_{index}.pkl" + logger.info(f"Loading the state of {get_pretty_name(obj)} from {load_location}") + obj.load_state_dict(load(load_location, map_location="cpu", weights_only=False)) diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__init__.py b/kbidu_env/Lib/site-packages/accelerate/commands/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c9cbe26c257b515f657c05e1996d517e69613972 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1c8534e064a2604a3ee44618f347accc6af1dea5 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/accelerate_cli.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/accelerate_cli.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9032d2c467a8f9b5e9b9aff6e50012b4fa1d5560 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/accelerate_cli.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/env.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/env.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4242d32f70836155d94e7833114f2cd4dc40011f Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/env.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/estimate.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/estimate.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d38eed94417fc81a3617dc40f6aea20c3a098c74 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/estimate.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/launch.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/launch.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6929669a06f335f45c62a3b5a91eec27360ac5b1 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/launch.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/merge.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/merge.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..072e1b7d115e544ceca06f32db496ccc6bef4e98 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/merge.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/test.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/test.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3ec7d999a744280b30b8f868876136ba7f435b8 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/test.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/to_fsdp2.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/to_fsdp2.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f7101911138732d49eff66277c81db7cb13972c Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/to_fsdp2.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/tpu.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/tpu.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2653ce30fb3b2f81501db0615fb873cd0004355c Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/tpu.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/utils.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f68e0265be75a88476cfb46c71b52ed402a419f Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/__pycache__/utils.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/accelerate_cli.py b/kbidu_env/Lib/site-packages/accelerate/commands/accelerate_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..b878c8debd874e1418b946775b11568c7487ad72 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/accelerate_cli.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from accelerate.commands.config import get_config_parser +from accelerate.commands.env import env_command_parser +from accelerate.commands.estimate import estimate_command_parser +from accelerate.commands.launch import launch_command_parser +from accelerate.commands.merge import merge_command_parser +from accelerate.commands.test import test_command_parser +from accelerate.commands.to_fsdp2 import to_fsdp2_command_parser +from accelerate.commands.tpu import tpu_command_parser +from accelerate.commands.utils import CustomArgumentParser + + +def main(): + parser = CustomArgumentParser("Accelerate CLI tool", usage="accelerate []", allow_abbrev=False) + subparsers = parser.add_subparsers(help="accelerate command helpers") + + # Register commands + get_config_parser(subparsers=subparsers) + estimate_command_parser(subparsers=subparsers) + env_command_parser(subparsers=subparsers) + launch_command_parser(subparsers=subparsers) + merge_command_parser(subparsers=subparsers) + tpu_command_parser(subparsers=subparsers) + test_command_parser(subparsers=subparsers) + to_fsdp2_command_parser(subparsers=subparsers) + + # Let's go + args = parser.parse_args() + + if not hasattr(args, "func"): + parser.print_help() + exit(1) + + # Run + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/__init__.py b/kbidu_env/Lib/site-packages/accelerate/commands/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..649a15888cccd070b3d4ca9a600457c6ad59d4d3 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/config/__init__.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from .config import config_command_parser +from .config_args import default_config_file, load_config_from_file # noqa: F401 +from .default import default_command_parser +from .update import update_command_parser + + +def get_config_parser(subparsers=None): + parent_parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) + # The main config parser + config_parser = config_command_parser(subparsers) + # The subparser to add commands to + subcommands = config_parser.add_subparsers(title="subcommands", dest="subcommand") + + # Then add other parsers with the parent parser + default_command_parser(subcommands, parents=[parent_parser]) + update_command_parser(subcommands, parents=[parent_parser]) + + return config_parser + + +def main(): + config_parser = get_config_parser() + args = config_parser.parse_args() + + if not hasattr(args, "func"): + config_parser.print_help() + exit(1) + + # Run + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4c1e2b14cfcf71ea4d5418a1cc5d61bea24d4c9d Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/cluster.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/cluster.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69f51691cf7a93a7d023aef42913f391089fa037 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/cluster.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/config.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ca7aba62ce736008e15d917781989d5f8370b66 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/config.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/config_args.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/config_args.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7172c385d357f42e4740719940d5d2dc053b09d Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/config_args.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/config_utils.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/config_utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f43dacc1407675bab1f0b03d9cefc79003e66ba Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/config_utils.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/default.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/default.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5ddb82596d954ad3a67579f2e5951016871fa42 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/default.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/sagemaker.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/sagemaker.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..422378e69a32602d19d464d52c8f03a24c703b5a Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/sagemaker.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/update.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/update.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6cbf6d6cc4254bc77b6d5a73fe54a05183e66301 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/config/__pycache__/update.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/cluster.py b/kbidu_env/Lib/site-packages/accelerate/commands/config/cluster.py new file mode 100644 index 0000000000000000000000000000000000000000..fce80efdcbbb6e36572716d38b3d74491ebc918f --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/config/cluster.py @@ -0,0 +1,939 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from ...utils import ( + ComputeEnvironment, + DistributedType, + is_deepspeed_available, + is_fp8_available, + is_hpu_available, + is_mlu_available, + is_mps_available, + is_msamp_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_torchao_available, + is_transformer_engine_available, + is_transformers_available, + is_xpu_available, +) +from ...utils.constants import ( + DEEPSPEED_MULTINODE_LAUNCHERS, + FSDP2_STATE_DICT_TYPE, + FSDP_AUTO_WRAP_POLICY, + FSDP_BACKWARD_PREFETCH, + FSDP_SHARDING_STRATEGY, + FSDP_STATE_DICT_TYPE, + TORCH_DYNAMO_MODES, +) +from .config_args import ClusterConfig +from .config_utils import ( + DYNAMO_BACKENDS, + _ask_field, + _ask_options, + _convert_distributed_mode, + _convert_dynamo_backend, + _convert_fp8_backend, + _convert_mixed_precision, + _convert_yes_no_to_bool, +) + + +def get_cluster_input(): + distributed_type = _ask_options( + "Which type of machine are you using?", + [ + "No distributed training", + "multi-CPU", + "multi-XPU", + "multi-HPU", + "multi-GPU", + "multi-NPU", + "multi-MLU", + "multi-SDAA", + "multi-MUSA", + "multi-NEURON", + "TPU", + ], + _convert_distributed_mode, + ) + + machine_rank = 0 + num_machines = 1 + num_processes = 1 + gpu_ids = None + main_process_ip = None + main_process_port = None + rdzv_backend = "static" + same_network = True + debug = False + + if distributed_type in [ + DistributedType.MULTI_GPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_CPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + ]: + num_machines = _ask_field( + "How many different machines will you use (use more than 1 for multi-node training)? [1]: ", + int, + default=1, + ) + if num_machines > 1: + machine_rank = _ask_options( + "What is the rank of this machine?", + list(range(num_machines)), + int, + ) + main_process_ip = _ask_field( + "What is the IP address of the machine that will host the main process? ", + ) + main_process_port = _ask_field( + "What is the port you will use to communicate with the main process? ", + int, + ) + same_network = _ask_field( + "Are all the machines on the same local network? Answer `no` if nodes are on the cloud and/or on different network hosts [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + error_message="Please enter yes or no.", + ) + if not same_network: + rdzv_backend = _ask_field( + "What rendezvous backend will you use? ('static', 'c10d', ...): ", default="static" + ) + debug = _ask_field( + "Should distributed operations be checked while running for errors? This can avoid timeout issues but will be slower. [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + if distributed_type == DistributedType.NO: + use_cpu = _ask_field( + "Do you want to run your training on CPU only (even if a GPU / Apple Silicon / Ascend NPU device is available)? [yes/NO]:", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + elif distributed_type == DistributedType.MULTI_CPU: + use_cpu = True + else: + use_cpu = False + + mpirun_config = {} + + if use_cpu: + if distributed_type == DistributedType.MULTI_CPU: + use_mpirun = _ask_field( + "Do you want accelerate to launch mpirun? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_mpirun: + mpirun_hostfile = _ask_field( + "Please enter the path to the hostfile to use with mpirun [~/hostfile]: ", + str, + default="~/hostfile", + ) + mpirun_config["mpirun_hostfile"] = os.path.expanduser(mpirun_hostfile.strip()) + + dynamo_config = {} + use_dynamo = _ask_field( + "Do you wish to optimize your script with torch dynamo?[yes/NO]:", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_dynamo: + prefix = "dynamo_" + dynamo_config[prefix + "backend"] = _ask_options( + "Which dynamo backend would you like to use?", + [x.lower() for x in DYNAMO_BACKENDS], + _convert_dynamo_backend, + default=2, + ) + use_custom_options = _ask_field( + "Do you want to customize the defaults sent to torch.compile? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + if use_custom_options: + dynamo_config[prefix + "mode"] = _ask_options( + "Which mode do you want to use?", + TORCH_DYNAMO_MODES, + lambda x: TORCH_DYNAMO_MODES[int(x)], + default=0, + ) + dynamo_config[prefix + "use_fullgraph"] = _ask_field( + "Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + dynamo_config[prefix + "use_dynamic"] = _ask_field( + "Do you want to enable dynamic shape tracing? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + dynamo_config[prefix + "use_regional_compilation"] = _ask_field( + "Do you want to enable regional compilation? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + use_mps = not use_cpu and is_mps_available() + deepspeed_config = {} + if ( + distributed_type + in [ + DistributedType.MULTI_GPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NEURON, + DistributedType.NO, + ] + and not use_mps + ): + use_deepspeed = _ask_field( + "Do you want to use DeepSpeed? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_deepspeed: + if distributed_type is DistributedType.MULTI_NEURON: + raise RuntimeError("DeepSpeed is not supported on Neuron devices.") + + distributed_type = DistributedType.DEEPSPEED + assert is_deepspeed_available(), ( + "DeepSpeed is not installed => run `pip3 install deepspeed` or build it from source" + ) + + if distributed_type == DistributedType.DEEPSPEED: + use_deepspeed_config = _ask_field( + "Do you want to specify a json file to a DeepSpeed config? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_deepspeed_config: + deepspeed_config["deepspeed_config_file"] = _ask_field( + "Please enter the path to the json DeepSpeed config file: ", + str, + default="none", + ) + else: + deepspeed_config["zero_stage"] = _ask_options( + "What should be your DeepSpeed's ZeRO optimization stage?", + [0, 1, 2, 3], + int, + default=2, + ) + + deepspeed_devices = ["none", "cpu", "nvme"] + if deepspeed_config["zero_stage"] >= 2: + deepspeed_config["offload_optimizer_device"] = _ask_options( + "Where to offload optimizer states?", deepspeed_devices, lambda x: deepspeed_devices[int(x)] + ) + deepspeed_config["offload_param_device"] = _ask_options( + "Where to offload parameters?", deepspeed_devices, lambda x: deepspeed_devices[int(x)] + ) + if deepspeed_config["offload_param_device"] == "nvme": + deepspeed_config["offload_param_nvme_path"] = _ask_field( + "Nvme Path to offload parameters?", + str, + default="/nvme", + ) + if deepspeed_config["offload_optimizer_device"] == "nvme": + deepspeed_config["offload_optimizer_nvme_path"] = _ask_field( + "Nvme Path to offload optimizer states?", + str, + default="/nvme", + ) + deepspeed_config["gradient_accumulation_steps"] = _ask_field( + "How many gradient accumulation steps you're passing in your script? [1]: ", + int, + default=1, + ) + use_gradient_clipping = _ask_field( + "Do you want to use gradient clipping? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_gradient_clipping: + deepspeed_config["gradient_clipping"] = _ask_field( + "What is the gradient clipping value? [1.0]: ", + float, + default=1.0, + ) + if deepspeed_config["zero_stage"] == 3: + deepspeed_config["zero3_save_16bit_model"] = _ask_field( + "Do you want to save 16-bit model weights when using ZeRO Stage-3? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + deepspeed_config["zero3_init_flag"] = _ask_field( + "Do you want to enable `deepspeed.zero.Init` when using ZeRO Stage-3 for constructing massive models? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if deepspeed_config["zero3_init_flag"]: + if not is_transformers_available(): + raise Exception( + "When `zero3_init_flag` is set, it requires Transformers to be installed. " + "Please run `pip3 install transformers`." + ) + use_moe = _ask_field( + "Do you want to enable Mixture-of-Experts training (MoE)? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_moe: + deepspeed_config["deepspeed_moe_layer_cls_names"] = _ask_field( + "Specify the comma-separated list of transformers MoE layer class names (case-sensitive), e.g : " + " `MixtralSparseMoeBlock`, `Qwen2MoeSparseMoeBlock`, `JetMoEAttention,JetMoEBlock` ... : ", + str, + ) + + if num_machines > 1: + launcher_query = "Which Type of launcher do you want to use?" + deepspeed_config["deepspeed_multinode_launcher"] = _ask_options( + launcher_query, + DEEPSPEED_MULTINODE_LAUNCHERS, + lambda x: DEEPSPEED_MULTINODE_LAUNCHERS[int(x)], + ) + + if deepspeed_config["deepspeed_multinode_launcher"] != DEEPSPEED_MULTINODE_LAUNCHERS[1]: + deepspeed_config["deepspeed_hostfile"] = _ask_field( + "DeepSpeed configures multi-node compute resources with hostfile. " + "Each row is of the format `hostname slots=[num_gpus]`, e.g., `localhost slots=2`; " + "for more information please refer official [documentation]" + "(https://www.deepspeed.ai/getting-started/#resource-configuration-multi-node). " + "Please specify the location of hostfile: ", + str, + ) + + is_exclusion_filter = _ask_field( + "Do you want to specify exclusion filter string? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if is_exclusion_filter: + deepspeed_config["deepspeed_exclusion_filter"] = _ask_field( + "DeepSpeed exclusion filter string: ", + str, + ) + + is_inclusion_filter = _ask_field( + "Do you want to specify inclusion filter string? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if is_inclusion_filter: + deepspeed_config["deepspeed_inclusion_filter"] = _ask_field( + "DeepSpeed inclusion filter string: ", + str, + ) + + fsdp_config = {} + + if distributed_type in [ + DistributedType.MULTI_GPU, + DistributedType.MULTI_NPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_XPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + ]: + use_fsdp = _ask_field( + "Do you want to use FullyShardedDataParallel? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_fsdp: + if distributed_type is DistributedType.MULTI_NEURON: + raise NotImplementedError("FSDP is not currently supported on Neuron devices.") + distributed_type = DistributedType.FSDP + + if distributed_type == DistributedType.FSDP: + fsdp_config["fsdp_version"] = _ask_options( + "What should be your FSDP version? [2]: ", + [1, 2], + lambda x: int(x) + 1, + default=1, + ) + fsdp_version = fsdp_config["fsdp_version"] # extract to a variable to simplify usage later + + if fsdp_version == 1: + sharding_strategy_query = "What should be your sharding strategy?" + fsdp_config["fsdp_reshard_after_forward"] = _ask_options( + sharding_strategy_query, + FSDP_SHARDING_STRATEGY, + lambda x: FSDP_SHARDING_STRATEGY[int(x)], + ) + else: + fsdp_config["fsdp_reshard_after_forward"] = _ask_field( + "Do you want to enable resharding after forward? [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + error_message="Please enter yes or no.", + ) + + fsdp_config["fsdp_offload_params"] = _ask_field( + "Do you want to offload parameters and gradients to CPU? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + fsdp_wrap_query = "What should be your auto wrap policy?" + fsdp_config["fsdp_auto_wrap_policy"] = _ask_options( + fsdp_wrap_query, + FSDP_AUTO_WRAP_POLICY, + lambda x: FSDP_AUTO_WRAP_POLICY[int(x)], + ) + if fsdp_config["fsdp_auto_wrap_policy"] == FSDP_AUTO_WRAP_POLICY[0]: + use_no_split_modules = _ask_field( + "Do you want to use the model's `_no_split_modules` to wrap. Only applicable for 🤗 Transformers [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if not use_no_split_modules: + fsdp_config["fsdp_transformer_layer_cls_to_wrap"] = _ask_field( + "Specify the comma-separated list of transformer layer class names (case-sensitive) to wrap ,e.g, :" + "`BertLayer`, `GPTJBlock`, `T5Block`, `BertLayer,BertEmbeddings,BertSelfOutput` ...? : ", + str, + ) + elif fsdp_config["fsdp_auto_wrap_policy"] == FSDP_AUTO_WRAP_POLICY[1]: + fsdp_config["fsdp_min_num_params"] = _ask_field( + "What should be your FSDP's minimum number of parameters for Default Auto Wrapping Policy? [1e8]: ", + int, + default=100000000, + ) + # Removed in FSDP2, ask for user input for FSDP1 + if fsdp_version == 1: + fsdp_backward_prefetch_query = "What should be your FSDP's backward prefetch policy?" + fsdp_config["fsdp_backward_prefetch"] = _ask_options( + fsdp_backward_prefetch_query, + FSDP_BACKWARD_PREFETCH, + lambda x: FSDP_BACKWARD_PREFETCH[int(x)], + ) + + fsdp_state_dict_type_query = "What should be your FSDP's state dict type?" + fsdp_config["fsdp_state_dict_type"] = _ask_options( + fsdp_state_dict_type_query, + FSDP_STATE_DICT_TYPE if fsdp_version == 1 else FSDP2_STATE_DICT_TYPE, + lambda x: FSDP_STATE_DICT_TYPE[int(x)] if fsdp_version == 1 else FSDP2_STATE_DICT_TYPE[int(x)], + default=0, + ) + # Not implemented in FSDP2, ask for user input for FSDP1 + if fsdp_version == 1: + fsdp_config["fsdp_forward_prefetch"] = _ask_field( + "Do you want to enable FSDP's forward prefetch policy? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + # Obsolete in FSDP2, ask for user input for FSDP1 + if fsdp_version == 1: + fsdp_config["fsdp_use_orig_params"] = _ask_field( + "Do you want to enable FSDP's `use_orig_params` feature? [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + error_message="Please enter yes or no.", + ) + fsdp_config["fsdp_cpu_ram_efficient_loading"] = _ask_field( + "Do you want to enable CPU RAM efficient model loading? Only applicable for 🤗 Transformers models. [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + error_message="Please enter yes or no.", + ) + # Obsolete in FSDP2, ask for user input for FSDP1 + if fsdp_version == 1: + if fsdp_config["fsdp_cpu_ram_efficient_loading"]: + fsdp_config["fsdp_sync_module_states"] = True + else: + fsdp_config["fsdp_sync_module_states"] = _ask_field( + "Do you want each individually wrapped FSDP unit to broadcast module parameters from rank 0 at the start? [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + error_message="Please enter yes or no.", + ) + fsdp_config["fsdp_activation_checkpointing"] = _ask_field( + "Do you want to enable FSDP activation checkpointing? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + parallelism_config = {} + + if fsdp_config.get("fsdp_version", 1) == 2: + use_parallelism_config = _ask_field( + "Do you want to use the parallelism config? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + if use_parallelism_config: + prefix = "parallelism_config_" + parallelism_config[prefix + "dp_replicate_size"] = _ask_field( + "What is the data parallelism replicate size? [1]: ", + int, + default=1, + error_message="Please enter an integer.", + ) + + parallelism_config[prefix + "dp_shard_size"] = _ask_field( + "What is the FSDP shard size? [1]: ", + int, + default=1, + error_message="Please enter an integer.", + ) + + parallelism_config[prefix + "tp_size"] = _ask_field( + "What is the tensor parallelism size? [1]: ", + int, + default=1, + error_message="Please enter an integer.", + ) + + parallelism_config[prefix + "cp_size"] = _ask_field( + "What is the context parallelism size? [1]: ", + int, + default=1, + error_message="Please enter an integer.", + ) + if parallelism_config[prefix + "cp_size"] > 1: + parallelism_config[prefix + "cp_comm_strategy"] = _ask_options( + "What is the compute parallelism communication strategy?", + ["allgather", "alltoall"], + lambda x: ["allgather", "alltoall"][int(x)], + default=0, + ) + + megatron_lm_config = {} + if distributed_type in [DistributedType.MULTI_GPU]: + use_megatron_lm = _ask_field( + "Do you want to use Megatron-LM ? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_megatron_lm: + distributed_type = DistributedType.MEGATRON_LM + if distributed_type == DistributedType.MEGATRON_LM: + prefix = "megatron_lm_" + megatron_lm_config[prefix + "tp_degree"] = _ask_field( + "What is the Tensor Parallelism degree/size? [1]:", + int, + default=1, + error_message="Please enter an integer.", + ) + if megatron_lm_config[prefix + "tp_degree"] > 1: + megatron_lm_config[prefix + "sequence_parallelism"] = _ask_field( + "Do you want to enable Sequence Parallelism? [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + error_message="Please enter yes or no.", + ) + + megatron_lm_config[prefix + "pp_degree"] = _ask_field( + "What is the Pipeline Parallelism degree/size? [1]:", + int, + default=1, + error_message="Please enter an integer.", + ) + if megatron_lm_config[prefix + "pp_degree"] > 1: + megatron_lm_config[prefix + "num_micro_batches"] = _ask_field( + "What is the number of micro-batches? [1]:", + int, + default=1, + error_message="Please enter an integer.", + ) + + megatron_lm_config[prefix + "recompute_activations"] = _ask_field( + "Do you want to enable selective activation recomputation? [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + error_message="Please enter yes or no.", + ) + + megatron_lm_config[prefix + "use_distributed_optimizer"] = _ask_field( + "Do you want to use distributed optimizer " + "which shards optimizer state and gradients across data parallel ranks? [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + error_message="Please enter yes or no.", + ) + + megatron_lm_config[prefix + "gradient_clipping"] = _ask_field( + "What is the gradient clipping value based on global L2 Norm (0 to disable)? [1.0]: ", + float, + default=1.0, + ) + # TPU specific defaults + tpu_commands = None + tpu_command_file = None + tpu_downcast_bf16 = "no" + tpu_env = [] + tpu_name = None + tpu_vm = None + tpu_zone = None + tpu_use_sudo = False + tpu_use_cluster = False + + if distributed_type in [ + DistributedType.MULTI_CPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_GPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NPU, + DistributedType.MULTI_NEURON, + DistributedType.XLA, + ]: + machine_type = str(distributed_type).split(".")[1].replace("MULTI_", "") + if machine_type in ["TPU", "NEURON"]: + machine_type += " cores" + elif machine_type == "CPU": + machine_type = "processes" + else: + machine_type += "(s)" + num_processes = _ask_field( + f"How many {machine_type} should be used for distributed training? [1]:", + int, + default=1, + error_message="Please enter an integer.", + ) + elif distributed_type in [DistributedType.FSDP, DistributedType.DEEPSPEED, DistributedType.MEGATRON_LM]: + num_processes = _ask_field( + "How many GPU(s) should be used for distributed training? [1]:", + int, + default=1, + error_message="Please enter an integer.", + ) + else: + num_processes = 1 + + if (distributed_type == DistributedType.MULTI_GPU) and (num_machines == 1) and (num_processes == 1): + raise ValueError( + f"Specified distributed type {distributed_type} but only using 1 GPU on a single machine. Please select `No distributed training` for the type of machine you are using." + ) + + if ( + distributed_type + in [ + DistributedType.MULTI_GPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + DistributedType.NO, + ] + and not use_cpu + and not use_mps + ): + if is_npu_available(): + machine_type = "NPU(s)" + elif is_mlu_available(): + machine_type = "MLU(s)" + elif is_sdaa_available(): + machine_type = "SDAA(s)" + elif is_musa_available(): + machine_type = "MUSA(s)" + elif is_xpu_available(): + machine_type = "XPU(s)" + elif is_hpu_available(): + machine_type = "HPU(s)" + elif is_neuron_available(): + machine_type = "Neuron cores" + else: + machine_type = "GPU(s)" + gpu_ids = _ask_field( + f"What {machine_type} (by id) should be used for training on this machine as a comma-separated list? [all]:", + default="all", + ) + + # CPU affinity is only supported on NVIDIA hardware for now + enable_cpu_affinity = False + if distributed_type in (DistributedType.NO, DistributedType.MULTI_GPU) and not use_cpu and not use_mps: + enable_cpu_affinity = _ask_field( + "Would you like to enable numa efficiency? (Currently only supported on NVIDIA hardware). [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + fp8_config = None + if distributed_type == DistributedType.XLA: + mixed_precision = "no" + main_training_function = _ask_field( + "What is the name of the function in your script that should be launched in all parallel scripts? [main]: ", + default="main", + ) + tpu_use_cluster = _ask_field( + "Are you using a TPU cluster? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if tpu_use_cluster: + tpu_name = _ask_field( + "What is the name of your TPU cluster? ", + default=None, + error_message="Please enter the name of your TPU cluster.", + ) + tpu_zone = _ask_field( + "What is the zone of your TPU cluster? ", + default=None, + error_message="Please enter the zone of your TPU cluster.", + ) + tpu_use_sudo = _ask_field( + "To run a python script in a TPU pod, should `sudo` be used? [yes/NO]: ", + default=False, + error_message="Please enter yes or no.", + ) + run_commands = _ask_field( + "Do you have code you wish to run on startup in each pod? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if run_commands: + use_command_file = _ask_field( + "Is this code located in a bash script? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_command_file: + tpu_command_file = _ask_field( + "What is the path to your bash script? ", + default=None, + error_message="Please enter the path to your bash script.", + ) + tpu_command_file = os.path.abspath(tpu_command_file) + else: + print("Please enter each command separately you wish to run on startup in each pod.") + tpu_commands = [] + another_command = True + while another_command: + tpu_commands.append( + _ask_field( + "Please enter a single command to be ran ", + default=None, + error_message="Please enter the commands you wish to run on startup in each pod as a single string.", + ) + ) + another_command = _ask_field( + "Do you wish to add another command? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + tpu_vm = _ask_field( + "If not using an instance group, what are the names of the Compute VM instances to be used, separated by a comma: ", + default="", + ).split(",") + tpu_env = _ask_field( + "What environment variables do you wish to set in each pod, separated by a comma: ", + default="", + ).split(",") + + else: + main_training_function = "main" + if distributed_type == DistributedType.DEEPSPEED and use_deepspeed_config: + mixed_precision = None + else: + mixed_precision = _ask_options( + "Do you wish to use mixed precision?", + ["no", "fp16", "bf16", "fp8"], + _convert_mixed_precision, + ) + if mixed_precision == "fp8": + if not is_fp8_available(): + raise ValueError( + "FP8 (either torchao, Transformer Engine or MSAMP) is not installed on this machine." + ) + fp8_config = {} + fp8_config["backend"] = _ask_options( + "Which FP8 backend do you want to use?", + ["ao", "te", "msamp"], + _convert_fp8_backend, + ) + if fp8_config["backend"] == "TE": + if not is_transformer_engine_available(): + raise ValueError("TransformersEngine was selected, but it is not installed on this machine.") + fp8_config["use_autocast_during_eval"] = _ask_field( + "Do you want to use FP8 autocast during eval mode? Generally better metrics are found when this is disabled [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + ) + fp8_config["margin"] = _ask_field( + "What margin should be used for gradient scaling? [0]: ", + int, + default=0, + ) + fp8_config["interval"] = _ask_field( + "What interval should be used for for how often the scaling factor is recomputed? [1]: ", + int, + default=1, + ) + fp8_config["fp8_format"] = _ask_options( + "Which weight format should be used?", + ["HYBRID", "E4M3", "E5M2"], + lambda i: ["HYBRID", "E4M3", "E5M2"][i], + default=0, + ) + fp8_config["amax_history_length"] = _ask_field( + "What length of history should be used for the amax scaling factor computation? [1024]: ", + int, + default=1024, + ) + fp8_config["amax_compute_algorithm"] = _ask_options( + "Which algorithm should be used for the amax scaling factor computation?", + ["max", "most_recent"], + lambda x: "max" if x == 0 else "most_recent", + default=0, + ) + fp8_config["override_linear_precision"] = _ask_field( + "Do you want to to execute `fprop`, `dgrad`, and `wgrad` GEMMS in higher precision? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + ) + if fp8_config["override_linear_precision"]: + fprop = _ask_field( + "Should `fprop` be executed in higher precision? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + ) + dgrad = _ask_field( + "Should `dgrad` be executed in higher precision? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + ) + wgrad = _ask_field( + "Should `wgrad` be executed in higher precision? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + ) + fp8_config["override_linear_precision"] = (fprop, dgrad, wgrad) + else: + fp8_config["override_linear_precision"] = (False, False, False) + + elif fp8_config["backend"] == "MSAMP": + if not is_msamp_available(): + raise ValueError("MSAMP was selected, but it is not installed on this machine.") + fp8_config["optimization_level"] = _ask_options( + "Which optimization level should be used?", + ["O1", "O2"], + lambda x: "O1" if x == 0 else "O2", + default=1, + ) + + elif fp8_config["backend"] == "AO": + if not is_torchao_available(): + raise ValueError("torchao was selected, but it is not installed on this machine.") + fp8_config["enable_fsdp_float8_all_gather"] = _ask_field( + "Do you want to enable FSDP2 float8 all gather? This is recommended for better performance if using FSDP2. [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + ) + fp8_config["pad_inner_dim"] = _ask_field( + "Do you want to pad the inner dimension of weight matrices before float8 matmuls? This is required for _scaled_mm which has strict alignment requirements. Note: padding may cause memory spikes. [YES/no]: ", + _convert_yes_no_to_bool, + default=True, + ) + + if use_dynamo and mixed_precision == "no" and not use_cpu: + print( + "Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts." + ) + + if distributed_type == DistributedType.XLA and mixed_precision == "bf16": + tpu_downcast_bf16 = _ask_field( + "Should `torch.float` be cast as `bfloat16` and `torch.double` remain `float32` on TPUs?", default="no" + ) + + return ClusterConfig( + compute_environment=ComputeEnvironment.LOCAL_MACHINE, + distributed_type=distributed_type, + num_processes=num_processes, + gpu_ids=gpu_ids, + mixed_precision=mixed_precision, + downcast_bf16=tpu_downcast_bf16, + machine_rank=machine_rank, + num_machines=num_machines, + main_process_ip=main_process_ip, + main_process_port=main_process_port, + main_training_function=main_training_function, + fp8_config=fp8_config, + deepspeed_config=deepspeed_config, + fsdp_config=fsdp_config, + parallelism_config=parallelism_config, + megatron_lm_config=megatron_lm_config, + mpirun_config=mpirun_config, + use_cpu=use_cpu, + rdzv_backend=rdzv_backend, + same_network=same_network, + commands=tpu_commands, + command_file=tpu_command_file, + tpu_env=tpu_env, + tpu_name=tpu_name, + tpu_vm=tpu_vm, + tpu_zone=tpu_zone, + tpu_use_sudo=tpu_use_sudo, + tpu_use_cluster=tpu_use_cluster, + dynamo_config=dynamo_config, + debug=debug, + enable_cpu_affinity=enable_cpu_affinity, + ) diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/config.py b/kbidu_env/Lib/site-packages/accelerate/commands/config/config.py new file mode 100644 index 0000000000000000000000000000000000000000..72414f2abe62d76bd5133f4b0ed99bf34133f6f6 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/config/config.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os + +from accelerate.utils import ComputeEnvironment + +from .cluster import get_cluster_input +from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file # noqa: F401 +from .config_utils import _ask_field, _ask_options, _convert_compute_environment # noqa: F401 +from .sagemaker import get_sagemaker_input + + +description = "Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine" + + +def get_user_input(): + compute_environment = _ask_options( + "In which compute environment are you running?", + ["This machine", "AWS (Amazon SageMaker)"], + _convert_compute_environment, + ) + if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER: + config = get_sagemaker_input() + else: + config = get_cluster_input() + return config + + +def config_command_parser(subparsers=None): + if subparsers is not None: + parser = subparsers.add_parser("config", description=description) + else: + parser = argparse.ArgumentParser("Accelerate config command", description=description) + + parser.add_argument( + "--config_file", + default=None, + help=( + "The path to use to store the config file. Will default to a file named default_config.yaml in the cache " + "location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have " + "such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed " + "with 'huggingface'." + ), + ) + + if subparsers is not None: + parser.set_defaults(func=config_command) + return parser + + +def config_command(args): + config = get_user_input() + if args.config_file is not None: + config_file = args.config_file + else: + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + config_file = default_yaml_config_file + + if config_file.endswith(".json"): + config.to_json_file(config_file) + else: + config.to_yaml_file(config_file) + print(f"accelerate configuration saved at {config_file}") + + +def main(): + parser = config_command_parser() + args = parser.parse_args() + config_command(args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/config_args.py b/kbidu_env/Lib/site-packages/accelerate/commands/config/config_args.py new file mode 100644 index 0000000000000000000000000000000000000000..9a50fade655da8e1ef766817e013185728c6b7bc --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/config/config_args.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from dataclasses import dataclass +from enum import Enum +from typing import Optional, Union + +import yaml + +from ...utils import ComputeEnvironment, DistributedType, SageMakerDistributedType +from ...utils.constants import SAGEMAKER_PYTHON_VERSION, SAGEMAKER_PYTORCH_VERSION, SAGEMAKER_TRANSFORMERS_VERSION + + +hf_cache_home = os.path.expanduser( + os.environ.get("HF_HOME", os.path.join(os.environ.get("XDG_CACHE_HOME", "~/.cache"), "huggingface")) +) +cache_dir = os.path.join(hf_cache_home, "accelerate") +default_json_config_file = os.path.join(cache_dir, "default_config.yaml") +default_yaml_config_file = os.path.join(cache_dir, "default_config.yaml") + +# For backward compatibility: the default config is the json one if it's the only existing file. +if os.path.isfile(default_yaml_config_file) or not os.path.isfile(default_json_config_file): + default_config_file = default_yaml_config_file +else: + default_config_file = default_json_config_file + + +def load_config_from_file(config_file): + if config_file is not None: + if not os.path.isfile(config_file): + raise FileNotFoundError( + f"The passed configuration file `{config_file}` does not exist. " + "Please pass an existing file to `accelerate launch`, or use the default one " + "created through `accelerate config` and run `accelerate launch` " + "without the `--config_file` argument." + ) + else: + config_file = default_config_file + with open(config_file, encoding="utf-8") as f: + if config_file.endswith(".json"): + if ( + json.load(f).get("compute_environment", ComputeEnvironment.LOCAL_MACHINE) + == ComputeEnvironment.LOCAL_MACHINE + ): + config_class = ClusterConfig + else: + config_class = SageMakerConfig + return config_class.from_json_file(json_file=config_file) + else: + if ( + yaml.safe_load(f).get("compute_environment", ComputeEnvironment.LOCAL_MACHINE) + == ComputeEnvironment.LOCAL_MACHINE + ): + config_class = ClusterConfig + else: + config_class = SageMakerConfig + return config_class.from_yaml_file(yaml_file=config_file) + + +@dataclass +class BaseConfig: + compute_environment: ComputeEnvironment + distributed_type: Union[DistributedType, SageMakerDistributedType] + mixed_precision: str + use_cpu: bool + debug: bool + + def to_dict(self): + result = self.__dict__ + # For serialization, it's best to convert Enums to strings (or their underlying value type). + + def _convert_enums(value): + if isinstance(value, Enum): + return value.value + if isinstance(value, dict): + if not bool(value): + return None + for key1, value1 in value.items(): + value[key1] = _convert_enums(value1) + return value + + for key, value in result.items(): + result[key] = _convert_enums(value) + result = {k: v for k, v in result.items() if v is not None} + return result + + @staticmethod + def process_config(config_dict): + """ + Processes `config_dict` and sets default values for any missing keys + """ + if "compute_environment" not in config_dict: + config_dict["compute_environment"] = ComputeEnvironment.LOCAL_MACHINE + if "distributed_type" not in config_dict: + raise ValueError("A `distributed_type` must be specified in the config file.") + if "num_processes" not in config_dict and config_dict["distributed_type"] == DistributedType.NO: + config_dict["num_processes"] = 1 + if "mixed_precision" not in config_dict: + config_dict["mixed_precision"] = "fp16" if ("fp16" in config_dict and config_dict["fp16"]) else None + if "fp16" in config_dict: # Convert the config to the new format. + del config_dict["fp16"] + if "dynamo_backend" in config_dict: # Convert the config to the new format. + dynamo_backend = config_dict.pop("dynamo_backend") + config_dict["dynamo_config"] = {} if dynamo_backend == "NO" else {"dynamo_backend": dynamo_backend} + if "use_cpu" not in config_dict: + config_dict["use_cpu"] = False + if "debug" not in config_dict: + config_dict["debug"] = False + if "enable_cpu_affinity" not in config_dict: + config_dict["enable_cpu_affinity"] = False + return config_dict + + @classmethod + def from_json_file(cls, json_file=None): + json_file = default_json_config_file if json_file is None else json_file + with open(json_file, encoding="utf-8") as f: + config_dict = json.load(f) + config_dict = cls.process_config(config_dict) + extra_keys = sorted(set(config_dict.keys()) - set(cls.__dataclass_fields__.keys())) + if len(extra_keys) > 0: + raise ValueError( + f"The config file at {json_file} had unknown keys ({extra_keys}), please try upgrading your `accelerate`" + " version or fix (and potentially remove) these keys from your config file." + ) + + return cls(**config_dict) + + def to_json_file(self, json_file): + with open(json_file, "w", encoding="utf-8") as f: + content = json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + f.write(content) + + @classmethod + def from_yaml_file(cls, yaml_file=None): + yaml_file = default_yaml_config_file if yaml_file is None else yaml_file + with open(yaml_file, encoding="utf-8") as f: + config_dict = yaml.safe_load(f) + config_dict = cls.process_config(config_dict) + extra_keys = sorted(set(config_dict.keys()) - set(cls.__dataclass_fields__.keys())) + if len(extra_keys) > 0: + raise ValueError( + f"The config file at {yaml_file} had unknown keys ({extra_keys}), please try upgrading your `accelerate`" + " version or fix (and potentially remove) these keys from your config file." + ) + return cls(**config_dict) + + def to_yaml_file(self, yaml_file): + with open(yaml_file, "w", encoding="utf-8") as f: + yaml.safe_dump(self.to_dict(), f) + + def __post_init__(self): + if isinstance(self.compute_environment, str): + self.compute_environment = ComputeEnvironment(self.compute_environment) + if isinstance(self.distributed_type, str): + if self.compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER: + self.distributed_type = SageMakerDistributedType(self.distributed_type) + else: + self.distributed_type = DistributedType(self.distributed_type) + if getattr(self, "dynamo_config", None) is None: + self.dynamo_config = {} + + +@dataclass +class ClusterConfig(BaseConfig): + num_processes: int = -1 # For instance if we use SLURM and the user manually passes it in + machine_rank: int = 0 + num_machines: int = 1 + gpu_ids: Optional[str] = None + main_process_ip: Optional[str] = None + main_process_port: Optional[int] = None + rdzv_backend: Optional[str] = "static" + same_network: Optional[bool] = False + main_training_function: str = "main" + enable_cpu_affinity: bool = False + + # args for FP8 training + fp8_config: Optional[dict] = None + # args for deepspeed_plugin + deepspeed_config: Optional[dict] = None + # args for fsdp + fsdp_config: Optional[dict] = None + # args for parallelism config + parallelism_config: Optional[dict] = None + # args for megatron_lm + megatron_lm_config: Optional[dict] = None + # args for mpirun + mpirun_config: Optional[dict] = None + # args for TPU + downcast_bf16: bool = False + + # args for TPU pods + tpu_name: Optional[str] = None + tpu_zone: Optional[str] = None + tpu_use_cluster: bool = False + tpu_use_sudo: bool = False + command_file: Optional[str] = None + commands: list[str] = None + tpu_vm: list[str] = None + tpu_env: list[str] = None + + # args for dynamo + dynamo_config: Optional[dict] = None + + def __post_init__(self): + if self.deepspeed_config is None: + self.deepspeed_config = {} + if self.fsdp_config is None: + self.fsdp_config = {} + if self.megatron_lm_config is None: + self.megatron_lm_config = {} + if self.mpirun_config is None: + self.mpirun_config = {} + if self.fp8_config is None: + self.fp8_config = {} + if self.parallelism_config is None: + self.parallelism_config = {} + return super().__post_init__() + + +@dataclass +class SageMakerConfig(BaseConfig): + ec2_instance_type: str + iam_role_name: str + image_uri: Optional[str] = None + profile: Optional[str] = None + region: str = "us-east-1" + num_machines: int = 1 + gpu_ids: str = "all" + base_job_name: str = f"accelerate-sagemaker-{num_machines}" + pytorch_version: str = SAGEMAKER_PYTORCH_VERSION + transformers_version: str = SAGEMAKER_TRANSFORMERS_VERSION + py_version: str = SAGEMAKER_PYTHON_VERSION + sagemaker_inputs_file: Optional[str] = None + sagemaker_metrics_file: Optional[str] = None + additional_args: Optional[dict] = None + dynamo_config: Optional[dict] = None + enable_cpu_affinity: bool = False diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/config_utils.py b/kbidu_env/Lib/site-packages/accelerate/commands/config/config_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d29bf4726a8205e1d01fb5f279ddbc7d7160b944 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/config/config_utils.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from ...utils.dataclasses import ( + ComputeEnvironment, + DistributedType, + DynamoBackend, + FP8BackendType, + PrecisionType, + SageMakerDistributedType, +) +from ..menu import BulletMenu + + +DYNAMO_BACKENDS = [ + "EAGER", + "AOT_EAGER", + "INDUCTOR", + "AOT_TS_NVFUSER", + "NVPRIMS_NVFUSER", + "CUDAGRAPHS", + "OFI", + "FX2TRT", + "ONNXRT", + "TENSORRT", + "AOT_TORCHXLA_TRACE_ONCE", + "TORHCHXLA_TRACE_ONCE", + "TVM", +] + + +def _ask_field(input_text, convert_value=None, default=None, error_message=None): + ask_again = True + while ask_again: + result = input(input_text) + try: + if default is not None and len(result) == 0: + return default + return convert_value(result) if convert_value is not None else result + except Exception: + if error_message is not None: + print(error_message) + + +def _ask_options(input_text, options=[], convert_value=None, default=0): + menu = BulletMenu(input_text, options) + result = menu.run(default_choice=default) + return convert_value(result) if convert_value is not None else result + + +def _convert_compute_environment(value): + value = int(value) + return ComputeEnvironment(["LOCAL_MACHINE", "AMAZON_SAGEMAKER"][value]) + + +def _convert_distributed_mode(value): + value = int(value) + return DistributedType( + [ + "NO", + "MULTI_CPU", + "MULTI_XPU", + "MULTI_HPU", + "MULTI_GPU", + "MULTI_NPU", + "MULTI_MLU", + "MULTI_SDAA", + "MULTI_MUSA", + "MULTI_NEURON", + "XLA", + ][value] + ) + + +def _convert_dynamo_backend(value): + value = int(value) + return DynamoBackend(DYNAMO_BACKENDS[value]).value + + +def _convert_mixed_precision(value): + value = int(value) + return PrecisionType(["no", "fp16", "bf16", "fp8"][value]) + + +def _convert_sagemaker_distributed_mode(value): + value = int(value) + return SageMakerDistributedType(["NO", "DATA_PARALLEL", "MODEL_PARALLEL"][value]) + + +def _convert_fp8_backend(value): + value = int(value) + return FP8BackendType(["AO", "TE", "MSAMP"][value]) + + +def _convert_yes_no_to_bool(value): + return {"yes": True, "no": False}[value.lower()] + + +class SubcommandHelpFormatter(argparse.RawDescriptionHelpFormatter): + """ + A custom formatter that will remove the usage line from the help message for subcommands. + """ + + def _format_usage(self, usage, actions, groups, prefix): + usage = super()._format_usage(usage, actions, groups, prefix) + usage = usage.replace(" [] ", "") + return usage diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/default.py b/kbidu_env/Lib/site-packages/accelerate/commands/config/default.py new file mode 100644 index 0000000000000000000000000000000000000000..c505d8be937a1b59151e58f9298ef920e9f06f25 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/config/default.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import torch + +from ...utils import ( + is_hpu_available, + is_mlu_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_xpu_available, +) +from .config_args import ClusterConfig, default_json_config_file +from .config_utils import SubcommandHelpFormatter + + +description = "Create a default config file for Accelerate with only a few flags set." + + +def write_basic_config(mixed_precision="no", save_location: str = default_json_config_file): + """ + Creates and saves a basic cluster config to be used on a local machine with potentially multiple GPUs. Will also + set CPU if it is a CPU-only machine. + + Args: + mixed_precision (`str`, *optional*, defaults to "no"): + Mixed Precision to use. Should be one of "no", "fp16", or "bf16" + save_location (`str`, *optional*, defaults to `default_json_config_file`): + Optional custom save location. Should be passed to `--config_file` when using `accelerate launch`. Default + location is inside the huggingface cache folder (`~/.cache/huggingface`) but can be overridden by setting + the `HF_HOME` environmental variable, followed by `accelerate/default_config.yaml`. + """ + path = Path(save_location) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + print( + f"Configuration already exists at {save_location}, will not override. Run `accelerate config` manually or pass a different `save_location`." + ) + return False + mixed_precision = mixed_precision.lower() + if mixed_precision not in ["no", "fp16", "bf16", "fp8"]: + raise ValueError( + f"`mixed_precision` should be one of 'no', 'fp16', 'bf16', or 'fp8'. Received {mixed_precision}" + ) + config = { + "compute_environment": "LOCAL_MACHINE", + "mixed_precision": mixed_precision, + } + if is_mlu_available(): + num_mlus = torch.mlu.device_count() + config["num_processes"] = num_mlus + config["use_cpu"] = False + if num_mlus > 1: + config["distributed_type"] = "MULTI_MLU" + else: + config["distributed_type"] = "NO" + if is_sdaa_available(): + num_sdaas = torch.sdaa.device_count() + config["num_processes"] = num_sdaas + config["use_cpu"] = False + if num_sdaas > 1: + config["distributed_type"] = "MULTI_SDAA" + else: + config["distributed_type"] = "NO" + elif is_musa_available(): + num_musas = torch.musa.device_count() + config["num_processes"] = num_musas + config["use_cpu"] = False + if num_musas > 1: + config["distributed_type"] = "MULTI_MUSA" + else: + config["distributed_type"] = "NO" + elif is_hpu_available(): + num_hpus = torch.hpu.device_count() + config["num_processes"] = num_hpus + config["use_cpu"] = False + if num_hpus > 1: + config["distributed_type"] = "MULTI_HPU" + else: + config["distributed_type"] = "NO" + elif torch.cuda.is_available(): + num_gpus = torch.cuda.device_count() + config["num_processes"] = num_gpus + config["use_cpu"] = False + if num_gpus > 1: + config["distributed_type"] = "MULTI_GPU" + else: + config["distributed_type"] = "NO" + elif is_xpu_available(): + num_xpus = torch.xpu.device_count() + config["num_processes"] = num_xpus + config["use_cpu"] = False + if num_xpus > 1: + config["distributed_type"] = "MULTI_XPU" + else: + config["distributed_type"] = "NO" + elif is_npu_available(): + num_npus = torch.npu.device_count() + config["num_processes"] = num_npus + config["use_cpu"] = False + if num_npus > 1: + config["distributed_type"] = "MULTI_NPU" + else: + config["distributed_type"] = "NO" + elif is_neuron_available(): + num_neuron_cores = torch.neuron.device_count() + config["num_processes"] = num_neuron_cores + config["use_cpu"] = False + if num_neuron_cores > 1: + config["distributed_type"] = "MULTI_NEURON" + else: + config["distributed_type"] = "NO" + else: + num_xpus = 0 + config["use_cpu"] = True + config["num_processes"] = 1 + config["distributed_type"] = "NO" + config["debug"] = False + config["enable_cpu_affinity"] = False + config = ClusterConfig(**config) + config.to_json_file(path) + return path + + +def default_command_parser(parser, parents): + parser = parser.add_parser("default", parents=parents, help=description, formatter_class=SubcommandHelpFormatter) + parser.add_argument( + "--config_file", + default=default_json_config_file, + help=( + "The path to use to store the config file. Will default to a file named default_config.yaml in the cache " + "location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have " + "such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed " + "with 'huggingface'." + ), + dest="save_location", + ) + + parser.add_argument( + "--mixed_precision", + choices=["no", "fp16", "bf16"], + type=str, + help="Whether or not to use mixed precision training. " + "Choose between FP16 and BF16 (bfloat16) training. " + "BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later.", + default="no", + ) + parser.set_defaults(func=default_config_command) + return parser + + +def default_config_command(args): + config_file = write_basic_config(args.mixed_precision, args.save_location) + if config_file: + print(f"accelerate configuration saved at {config_file}") diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/sagemaker.py b/kbidu_env/Lib/site-packages/accelerate/commands/config/sagemaker.py new file mode 100644 index 0000000000000000000000000000000000000000..5092ef31fc4715f901be6c1e7bfe80c0b140d767 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/config/sagemaker.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os + +from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES +from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType +from ...utils.imports import is_boto3_available +from .config_args import SageMakerConfig +from .config_utils import ( + DYNAMO_BACKENDS, + _ask_field, + _ask_options, + _convert_dynamo_backend, + _convert_mixed_precision, + _convert_sagemaker_distributed_mode, + _convert_yes_no_to_bool, +) + + +if is_boto3_available(): + import boto3 # noqa: F401 + + +def _create_iam_role_for_sagemaker(role_name): + iam_client = boto3.client("iam") + + sagemaker_trust_policy = { + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Allow", "Principal": {"Service": "sagemaker.amazonaws.com"}, "Action": "sts:AssumeRole"} + ], + } + try: + # create the role, associated with the chosen trust policy + iam_client.create_role( + RoleName=role_name, AssumeRolePolicyDocument=json.dumps(sagemaker_trust_policy, indent=2) + ) + policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sagemaker:*", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:GetAuthorizationToken", + "cloudwatch:PutMetricData", + "cloudwatch:GetMetricData", + "cloudwatch:GetMetricStatistics", + "cloudwatch:ListMetrics", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + "logs:GetLogEvents", + "s3:CreateBucket", + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:PutObject", + ], + "Resource": "*", + } + ], + } + # attach policy to role + iam_client.put_role_policy( + RoleName=role_name, + PolicyName=f"{role_name}_policy_permission", + PolicyDocument=json.dumps(policy_document, indent=2), + ) + except iam_client.exceptions.EntityAlreadyExistsException: + print(f"role {role_name} already exists. Using existing one") + + +def _get_iam_role_arn(role_name): + iam_client = boto3.client("iam") + return iam_client.get_role(RoleName=role_name)["Role"]["Arn"] + + +def get_sagemaker_input(): + credentials_configuration = _ask_options( + "How do you want to authorize?", + ["AWS Profile", "Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) "], + int, + ) + aws_profile = None + if credentials_configuration == 0: + aws_profile = _ask_field("Enter your AWS Profile name: [default] ", default="default") + os.environ["AWS_PROFILE"] = aws_profile + else: + print( + "Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with," + "`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`" + ) + aws_access_key_id = _ask_field("AWS Access Key ID: ") + os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id + + aws_secret_access_key = _ask_field("AWS Secret Access Key: ") + os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key + + aws_region = _ask_field("Enter your AWS Region: [us-east-1]", default="us-east-1") + os.environ["AWS_DEFAULT_REGION"] = aws_region + + role_management = _ask_options( + "Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?", + ["Provide IAM Role name", "Create new IAM role using credentials"], + int, + ) + if role_management == 0: + iam_role_name = _ask_field("Enter your IAM role name: ") + else: + iam_role_name = "accelerate_sagemaker_execution_role" + print(f'Accelerate will create an iam role "{iam_role_name}" using the provided credentials') + _create_iam_role_for_sagemaker(iam_role_name) + + is_custom_docker_image = _ask_field( + "Do you want to use custom Docker image? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + docker_image = None + if is_custom_docker_image: + docker_image = _ask_field("Enter your Docker image: ", lambda x: str(x).lower()) + + is_sagemaker_inputs_enabled = _ask_field( + "Do you want to provide SageMaker input channels with data locations? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + sagemaker_inputs_file = None + if is_sagemaker_inputs_enabled: + sagemaker_inputs_file = _ask_field( + "Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): ", + lambda x: str(x).lower(), + ) + + is_sagemaker_metrics_enabled = _ask_field( + "Do you want to enable SageMaker metrics? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + sagemaker_metrics_file = None + if is_sagemaker_metrics_enabled: + sagemaker_metrics_file = _ask_field( + "Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): ", + lambda x: str(x).lower(), + ) + + distributed_type = _ask_options( + "What is the distributed mode?", + ["No distributed training", "Data parallelism"], + _convert_sagemaker_distributed_mode, + ) + dynamo_config = {} + use_dynamo = _ask_field( + "Do you wish to optimize your script with torch dynamo?[yes/NO]:", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + if use_dynamo: + prefix = "dynamo_" + dynamo_config[prefix + "backend"] = _ask_options( + "Which dynamo backend would you like to use?", + [x.lower() for x in DYNAMO_BACKENDS], + _convert_dynamo_backend, + default=2, + ) + use_custom_options = _ask_field( + "Do you want to customize the defaults sent to torch.compile? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + if use_custom_options: + dynamo_config[prefix + "mode"] = _ask_options( + "Which mode do you want to use?", + TORCH_DYNAMO_MODES, + lambda x: TORCH_DYNAMO_MODES[int(x)], + default="default", + ) + dynamo_config[prefix + "use_fullgraph"] = _ask_field( + "Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + dynamo_config[prefix + "use_dynamic"] = _ask_field( + "Do you want to enable dynamic shape tracing? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + dynamo_config[prefix + "use_regional_compilation"] = _ask_field( + "Do you want to enable regional compilation? [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + ec2_instance_query = "Which EC2 instance type you want to use for your training?" + if distributed_type != SageMakerDistributedType.NO: + ec2_instance_type = _ask_options( + ec2_instance_query, SAGEMAKER_PARALLEL_EC2_INSTANCES, lambda x: SAGEMAKER_PARALLEL_EC2_INSTANCES[int(x)] + ) + else: + ec2_instance_query += "? [ml.p3.2xlarge]:" + ec2_instance_type = _ask_field(ec2_instance_query, lambda x: str(x).lower(), default="ml.p3.2xlarge") + + debug = False + if distributed_type != SageMakerDistributedType.NO: + debug = _ask_field( + "Should distributed operations be checked while running for errors? This can avoid timeout issues but will be slower. [yes/NO]: ", + _convert_yes_no_to_bool, + default=False, + error_message="Please enter yes or no.", + ) + + num_machines = 1 + if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL): + num_machines = _ask_field( + "How many machines do you want use? [1]: ", + int, + default=1, + ) + + mixed_precision = _ask_options( + "Do you wish to use FP16 or BF16 (mixed precision)?", + ["no", "fp16", "bf16", "fp8"], + _convert_mixed_precision, + ) + + if use_dynamo and mixed_precision == "no": + print( + "Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts." + ) + + return SageMakerConfig( + image_uri=docker_image, + compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER, + distributed_type=distributed_type, + use_cpu=False, + dynamo_config=dynamo_config, + ec2_instance_type=ec2_instance_type, + profile=aws_profile, + region=aws_region, + iam_role_name=iam_role_name, + mixed_precision=mixed_precision, + num_machines=num_machines, + sagemaker_inputs_file=sagemaker_inputs_file, + sagemaker_metrics_file=sagemaker_metrics_file, + debug=debug, + ) diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/config/update.py b/kbidu_env/Lib/site-packages/accelerate/commands/config/update.py new file mode 100644 index 0000000000000000000000000000000000000000..369bb638e60516cb1ef46dbe53497c915834cbe4 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/config/update.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from .config_args import default_config_file, load_config_from_file +from .config_utils import SubcommandHelpFormatter + + +description = "Update an existing config file with the latest defaults while maintaining the old configuration." + + +def update_config(args): + """ + Update an existing config file with the latest defaults while maintaining the old configuration. + """ + config_file = args.config_file + if config_file is None and Path(default_config_file).exists(): + config_file = default_config_file + elif not Path(config_file).exists(): + raise ValueError(f"The passed config file located at {config_file} doesn't exist.") + config = load_config_from_file(config_file) + + if config_file.endswith(".json"): + config.to_json_file(config_file) + else: + config.to_yaml_file(config_file) + return config_file + + +def update_command_parser(parser, parents): + parser = parser.add_parser("update", parents=parents, help=description, formatter_class=SubcommandHelpFormatter) + parser.add_argument( + "--config_file", + default=None, + help=( + "The path to the config file to update. Will default to a file named default_config.yaml in the cache " + "location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have " + "such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed " + "with 'huggingface'." + ), + ) + + parser.set_defaults(func=update_config_command) + return parser + + +def update_config_command(args): + config_file = update_config(args) + print(f"Successfully updated the configuration file at {config_file}.") diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/env.py b/kbidu_env/Lib/site-packages/accelerate/commands/env.py new file mode 100644 index 0000000000000000000000000000000000000000..22790c40bfb7c489dda7e5e5aa172998f80fd45b --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/env.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python + +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import platform +import subprocess + +import numpy as np +import psutil +import torch + +from accelerate import __version__ as version +from accelerate.commands.config import default_config_file, load_config_from_file + +from ..utils import ( + is_mlu_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_xpu_available, +) + + +def env_command_parser(subparsers=None): + if subparsers is not None: + parser = subparsers.add_parser("env") + else: + parser = argparse.ArgumentParser("Accelerate env command") + + parser.add_argument( + "--config_file", default=None, help="The config file to use for the default values in the launching script." + ) + + if subparsers is not None: + parser.set_defaults(func=env_command) + return parser + + +def env_command(args): + pt_version = torch.__version__ + pt_cuda_available = torch.cuda.is_available() + pt_xpu_available = is_xpu_available() + pt_mlu_available = is_mlu_available() + pt_sdaa_available = is_sdaa_available() + pt_musa_available = is_musa_available() + pt_npu_available = is_npu_available() + pt_neuron_available = is_neuron_available() + + accelerator = "N/A" + if pt_cuda_available: + accelerator = "CUDA" + elif pt_xpu_available: + accelerator = "XPU" + elif pt_mlu_available: + accelerator = "MLU" + elif pt_sdaa_available: + accelerator = "SDAA" + elif pt_musa_available: + accelerator = "MUSA" + elif pt_npu_available: + accelerator = "NPU" + elif pt_neuron_available: + accelerator = "NEURON" + + accelerate_config = "Not found" + # Get the default from the config file. + if args.config_file is not None or os.path.isfile(default_config_file): + accelerate_config = load_config_from_file(args.config_file).to_dict() + + # if we can run which, get it + command = None + bash_location = "Not found" + if os.name == "nt": + command = ["where", "accelerate"] + elif os.name == "posix": + command = ["which", "accelerate"] + if command is not None: + bash_location = subprocess.check_output(command, text=True, stderr=subprocess.STDOUT).strip() + info = { + "`Accelerate` version": version, + "Platform": platform.platform(), + "`accelerate` bash location": bash_location, + "Python version": platform.python_version(), + "Numpy version": np.__version__, + "PyTorch version": f"{pt_version}", + "PyTorch accelerator": accelerator, + "System RAM": f"{psutil.virtual_memory().total / 1024**3:.2f} GB", + } + if pt_cuda_available: + info["GPU type"] = torch.cuda.get_device_name() + elif pt_xpu_available: + info["XPU type"] = torch.xpu.get_device_name() + elif pt_mlu_available: + info["MLU type"] = torch.mlu.get_device_name() + elif pt_sdaa_available: + info["SDAA type"] = torch.sdaa.get_device_name() + elif pt_musa_available: + info["MUSA type"] = torch.musa.get_device_name() + elif pt_neuron_available: + info["NEURON type"] = torch.neuron.get_device_name() + elif pt_npu_available: + info["CANN version"] = torch.version.cann + + print("\nCopy-and-paste the text below in your GitHub issue\n") + print("\n".join([f"- {prop}: {val}" for prop, val in info.items()])) + + print("- `Accelerate` default config:" if args.config_file is None else "- `Accelerate` config passed:") + accelerate_config_str = ( + "\n".join([f"\t- {prop}: {val}" for prop, val in accelerate_config.items()]) + if isinstance(accelerate_config, dict) + else f"\t{accelerate_config}" + ) + print(accelerate_config_str) + + info["`Accelerate` configs"] = accelerate_config + + return info + + +def main() -> int: + parser = env_command_parser() + args = parser.parse_args() + env_command(args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/estimate.py b/kbidu_env/Lib/site-packages/accelerate/commands/estimate.py new file mode 100644 index 0000000000000000000000000000000000000000..c1a8b41cead9d0d309bf482bde5a8f558b18b6fd --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/estimate.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python + +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional + +import torch +from huggingface_hub import model_info +from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError + +from accelerate import init_empty_weights +from accelerate.commands.utils import CustomArgumentParser +from accelerate.utils import ( + calculate_maximum_sizes, + convert_bytes, + is_timm_available, + is_transformers_available, +) + + +if is_transformers_available(): + import transformers + from transformers import AutoConfig, AutoModel + +if is_timm_available(): + import timm + + +def verify_on_hub(repo: str, token: Optional[str] = None): + "Verifies that the model is on the hub and returns the model info." + try: + return model_info(repo, token=token) + except (OSError, GatedRepoError): + return "gated" + except RepositoryNotFoundError: + return "repo" + + +def check_has_model(error): + """ + Checks what library spawned `error` when a model is not found + """ + if is_timm_available() and isinstance(error, RuntimeError) and "Unknown model" in error.args[0]: + return "timm" + elif ( + is_transformers_available() + and isinstance(error, OSError) + and "does not appear to have a file named" in error.args[0] + ): + return "transformers" + else: + return "unknown" + + +def create_empty_model( + model_name: str, library_name: str, trust_remote_code: bool = False, access_token: Optional[str] = None +): + """ + Creates an empty model in full precision from its parent library on the `Hub` to calculate the overall memory + consumption. + + Args: + model_name (`str`): + The model name on the Hub + library_name (`str`): + The library the model has an integration with, such as `transformers`. Will be used if `model_name` has no + metadata on the Hub to determine the library. + trust_remote_code (`bool`, `optional`, defaults to `False`): + Whether or not to allow for custom models defined on the Hub in their own modeling files. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + access_token (`str`, `optional`, defaults to `None`): + The access token to use to access private or gated models on the Hub. (for use on the Gradio app) + + Returns: + `torch.nn.Module`: The torch model that has been initialized on the `meta` device. + + """ + model_info = verify_on_hub(model_name, access_token) + # Simplified errors + if model_info == "gated": + raise OSError( + f"Repo for model `{model_name}` is gated. You must be authenticated to access it. Please run `huggingface-cli login`." + ) + elif model_info == "repo": + raise OSError( + f"Repo for model `{model_name}` does not exist on the Hub. If you are trying to access a private repo," + " make sure you are authenticated via `huggingface-cli login` and have access." + ) + if library_name is None: + library_name = getattr(model_info, "library_name", False) + if not library_name: + raise ValueError( + f"Model `{model_name}` does not have any library metadata on the Hub, please manually pass in a `--library_name` to use (such as `transformers`)" + ) + if library_name == "transformers": + if not is_transformers_available(): + raise ImportError( + f"To check `{model_name}`, `transformers` must be installed. Please install it via `pip install transformers`" + ) + print(f"Loading pretrained config for `{model_name}` from `transformers`...") + if model_info.config is None: + raise RuntimeError(f"Tried to load `{model_name}` with `transformers` but it does not have any metadata.") + + auto_map = model_info.config.get("auto_map", False) + config = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code, token=access_token) + with init_empty_weights(): + # remote code could specify a specific `AutoModel` class in the `auto_map` + constructor = AutoModel + if isinstance(auto_map, dict): + value = None + for key in auto_map.keys(): + if key.startswith("AutoModelFor"): + value = key + break + if value is not None: + constructor = getattr(transformers, value) + # we need to pass the dtype, otherwise it is going to use the torch_dtype that is saved in the config + model = constructor.from_config(config, torch_dtype=torch.float32, trust_remote_code=trust_remote_code) + elif library_name == "timm": + if not is_timm_available(): + raise ImportError( + f"To check `{model_name}`, `timm` must be installed. Please install it via `pip install timm`" + ) + print(f"Loading pretrained config for `{model_name}` from `timm`...") + with init_empty_weights(): + model = timm.create_model(model_name, pretrained=False) + else: + raise ValueError( + f"Library `{library_name}` is not supported yet, please open an issue on GitHub for us to add support." + ) + return model + + +def create_ascii_table(headers: list, rows: list, title: str): + "Creates a pretty table from a list of rows, minimal version of `tabulate`." + sep_char, in_between = "│", "─" + column_widths = [] + for i in range(len(headers)): + column_values = [row[i] for row in rows] + [headers[i]] + max_column_width = max(len(value) for value in column_values) + column_widths.append(max_column_width) + + formats = [f"%{column_widths[i]}s" for i in range(len(rows[0]))] + + pattern = f"{sep_char}{sep_char.join(formats)}{sep_char}" + diff = 0 + + def make_row(left_char, middle_char, right_char): + return f"{left_char}{middle_char.join([in_between * n for n in column_widths])}{in_between * diff}{right_char}" + + separator = make_row("├", "┼", "┤") + if len(title) > sum(column_widths): + diff = abs(len(title) - len(separator)) + column_widths[-1] += diff + + # Update with diff + separator = make_row("├", "┼", "┤") + initial_rows = [ + make_row("┌", in_between, "┐"), + f"{sep_char}{title.center(len(separator) - 2)}{sep_char}", + make_row("├", "┬", "┤"), + ] + table = "\n".join(initial_rows) + "\n" + column_widths[-1] += diff + centered_line = [text.center(column_widths[i]) for i, text in enumerate(headers)] + table += f"{pattern % tuple(centered_line)}\n{separator}\n" + for i, line in enumerate(rows): + centered_line = [t.center(column_widths[i]) for i, t in enumerate(line)] + table += f"{pattern % tuple(centered_line)}\n" + table += f"└{'┴'.join([in_between * n for n in column_widths])}┘" + + return table + + +def estimate_command_parser(subparsers=None): + if subparsers is not None: + parser = subparsers.add_parser("estimate-memory") + else: + parser = CustomArgumentParser( + description="Model size estimator for fitting a model onto device(e.g. cuda, xpu) memory." + ) + + parser.add_argument("model_name", type=str, help="The model name on the Hugging Face Hub.") + parser.add_argument( + "--library_name", + type=str, + help="The library the model has an integration with, such as `transformers`, needed only if this information is not stored on the Hub.", + choices=["timm", "transformers"], + ) + parser.add_argument( + "--dtypes", + type=str, + nargs="+", + default=["float32", "float16", "int8", "int4"], + help="The dtypes to use for the model, must be one (or many) of `float32`, `float16`, `int8`, and `int4`", + choices=["float32", "float16", "int8", "int4"], + ) + parser.add_argument( + "--trust_remote_code", + action="store_true", + help="""Whether or not to allow for custom models defined on the Hub in their own modeling files. This flag + should only be used for repositories you trust and in which you have read the code, as it will execute + code present on the Hub on your local machine.""", + default=False, + ) + + if subparsers is not None: + parser.set_defaults(func=estimate_command) + return parser + + +def estimate_training_usage(bytes: int, mixed_precision: str, msamp_config: Optional[str] = None) -> dict: + """ + Given an amount of `bytes` and `mixed_precision`, calculates how much training memory is needed for a batch size of + 1. + + Args: + bytes (`int`): + The size of the model being trained. + mixed_precision (`str`): + The mixed precision that would be ran. + msamp_config (`str`): + The msamp config to estimate the training memory for if `mixed_precision` is set to `"fp8"`. + """ + memory_sizes = {"model": -1, "optimizer": -1, "gradients": -1, "step": -1} + fp32_size = bytes + fp16_size = bytes // 2 + + if mixed_precision == "float32": + memory_sizes["model"] = fp32_size + memory_sizes["gradients"] = fp32_size + memory_sizes["optimizer"] = fp32_size * 2 + memory_sizes["step"] = fp32_size * 4 + elif mixed_precision in ("float16", "bfloat16") or (mixed_precision == "fp8" and msamp_config is None): + # With native `TransformersEngine`, there is no memory savings with FP8 + # With mixed precision training, the model has weights stored + # in FP16 and FP32 + memory_sizes["model"] = fp32_size + # 1.5 from weight gradient + computation (GEMM) + memory_sizes["gradients"] = fp32_size + fp16_size + # 2x from optimizer states + memory_sizes["optimizer"] = fp32_size * 2 # Optimizer states + memory_sizes["step"] = memory_sizes["optimizer"] + return memory_sizes + + +def gather_data(args): + "Creates an empty model and gathers the data for the sizes" + try: + model = create_empty_model( + args.model_name, library_name=args.library_name, trust_remote_code=args.trust_remote_code + ) + except (RuntimeError, OSError) as e: + library = check_has_model(e) + if library != "unknown": + raise RuntimeError( + f"Tried to load `{args.model_name}` with `{library}` but a possible model to load was not found inside the repo." + ) + raise e + + total_size, largest_layer = calculate_maximum_sizes(model) + + data = [] + + for dtype in args.dtypes: + dtype_total_size = total_size + dtype_largest_layer = largest_layer[0] + dtype_training_size = estimate_training_usage(dtype_total_size, dtype) + if dtype == "float16": + dtype_total_size /= 2 + dtype_largest_layer /= 2 + elif dtype == "int8": + dtype_total_size /= 4 + dtype_largest_layer /= 4 + elif dtype == "int4": + dtype_total_size /= 8 + dtype_largest_layer /= 8 + data.append([dtype, dtype_largest_layer, dtype_total_size, dtype_training_size]) + return data + + +def estimate_command(args): + data = gather_data(args) + for row in data: + for i, item in enumerate(row): + if isinstance(item, (int, float)): + row[i] = convert_bytes(item) + elif isinstance(item, dict): + training_usage = max(item.values()) + row[i] = convert_bytes(training_usage) if training_usage != -1 else "N/A" + + headers = ["dtype", "Largest Layer", "Total Size", "Training using Adam"] + + title = f"Memory Usage for loading `{args.model_name}`" + table = create_ascii_table(headers, data, title) + print(table) + + +def main(): + parser = estimate_command_parser() + args = parser.parse_args() + estimate_command(args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/launch.py b/kbidu_env/Lib/site-packages/accelerate/commands/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..1cb324461a8d4052714cfecc386712957a19199d --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/launch.py @@ -0,0 +1,1417 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import importlib +import logging +import os +import subprocess +import sys +from pathlib import Path + +import torch + +from accelerate.commands.config import default_config_file, load_config_from_file +from accelerate.commands.config.config_args import SageMakerConfig +from accelerate.commands.config.config_utils import DYNAMO_BACKENDS +from accelerate.commands.utils import CustomArgumentParser +from accelerate.state import get_int_from_env +from accelerate.utils import ( + ComputeEnvironment, + DistributedType, + PrepareForLaunch, + _filter_args, + check_cuda_p2p_ib_support, + convert_dict_to_env_variables, + is_bf16_available, + is_deepspeed_available, + is_hpu_available, + is_mlu_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_rich_available, + is_sagemaker_available, + is_sdaa_available, + is_torch_xla_available, + is_xpu_available, + patch_environment, + prepare_deepspeed_cmd_env, + prepare_multi_gpu_env, + prepare_sagemager_args_inputs, + prepare_simple_launcher_cmd_env, + prepare_tpu, + str_to_bool, +) +from accelerate.utils.constants import DEEPSPEED_MULTINODE_LAUNCHERS, TORCH_DYNAMO_MODES + + +if is_rich_available(): + from rich import get_console + from rich.logging import RichHandler + + FORMAT = "%(message)s" + logging.basicConfig(format=FORMAT, datefmt="[%X]", handlers=[RichHandler()]) + + +logger = logging.getLogger(__name__) + + +options_to_group = { + "multi_gpu": "Distributed GPUs", + "tpu": "TPU", + "use_deepspeed": "DeepSpeed Arguments", + "use_fsdp": "FSDP Arguments", + "use_megatron_lm": "Megatron-LM Arguments", + "fp8_backend": "FP8 Arguments", +} + + +def clean_option(option): + "Finds all cases of - after the first two characters and changes them to _" + if "fp8_backend" in option: + option = "--fp8_backend" + if option.startswith("--"): + return option[2:].replace("-", "_") + + +class CustomHelpFormatter(argparse.HelpFormatter): + """ + This is a custom help formatter that will hide all arguments that are not used in the command line when the help is + called. This is useful for the case where the user is using a specific platform and only wants to see the arguments + for that platform. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.titles = [ + "Hardware Selection Arguments", + "Resource Selection Arguments", + "Training Paradigm Arguments", + "positional arguments", + "optional arguments", + ] + + def add_argument(self, action: argparse.Action): + if "accelerate" in sys.argv[0] and "launch" in sys.argv[1:]: + args = sys.argv[2:] + else: + args = sys.argv[1:] + + if len(args) > 1: + args = list(map(clean_option, args)) + used_platforms = [arg for arg in args if arg in options_to_group.keys()] + used_titles = [options_to_group[o] for o in used_platforms] + if action.container.title not in self.titles + used_titles: + action.help = argparse.SUPPRESS + elif action.container.title == "Hardware Selection Arguments": + if set(action.option_strings).isdisjoint(set(args)): + action.help = argparse.SUPPRESS + else: + action.help = action.help + " (currently selected)" + elif action.container.title == "Training Paradigm Arguments": + if set(action.option_strings).isdisjoint(set(args)): + action.help = argparse.SUPPRESS + else: + action.help = action.help + " (currently selected)" + + action.option_strings = [s for s in action.option_strings if "-" not in s[2:]] + super().add_argument(action) + + def end_section(self): + if len(self._current_section.items) < 2: + self._current_section.items = [] + self._current_section.heading = "" + super().end_section() + + +def launch_command_parser(subparsers=None): + description = "Launch a python script in a distributed scenario. Arguments can be passed in with either hyphens (`--num-processes=2`) or underscores (`--num_processes=2`)" + if subparsers is not None: + parser = subparsers.add_parser( + "launch", description=description, add_help=False, allow_abbrev=False, formatter_class=CustomHelpFormatter + ) + else: + parser = CustomArgumentParser( + "Accelerate launch command", + description=description, + add_help=False, + allow_abbrev=False, + formatter_class=CustomHelpFormatter, + ) + + parser.add_argument("-h", "--help", action="help", help="Show this help message and exit.") + + parser.add_argument( + "--config_file", + default=None, + help="The config file to use for the default values in the launching script.", + ) + parser.add_argument( + "--quiet", + "-q", + action="store_true", + help="Silence subprocess errors from the launch stack trace and only show the relevant tracebacks. (Only applicable to DeepSpeed and single-process configurations)", + ) + # Hardware selection arguments + hardware_args = parser.add_argument_group( + "Hardware Selection Arguments", "Arguments for selecting the hardware to be used." + ) + hardware_args.add_argument( + "--cpu", default=False, action="store_true", help="Whether or not to force the training on the CPU." + ) + hardware_args.add_argument( + "--multi_gpu", + default=False, + action="store_true", + help="Whether or not this should launch a distributed GPU training.", + ) + hardware_args.add_argument( + "--tpu", default=False, action="store_true", help="Whether or not this should launch a TPU training." + ) + # Resource selection arguments + resource_args = parser.add_argument_group( + "Resource Selection Arguments", "Arguments for fine-tuning how available hardware should be used." + ) + resource_args.add_argument( + "--mixed_precision", + type=str, + choices=["no", "fp16", "bf16", "fp8"], + help="Whether or not to use mixed precision training. " + "Choose between FP16 and BF16 (bfloat16) training. " + "BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later.", + ) + resource_args.add_argument( + "--num_processes", type=int, default=None, help="The total number of processes to be launched in parallel." + ) + resource_args.add_argument( + "--num_machines", type=int, default=None, help="The total number of machines used in this training." + ) + resource_args.add_argument( + "--num_cpu_threads_per_process", + type=int, + default=None, + help="The number of CPU threads per process. Can be tuned for optimal performance.", + ) + resource_args.add_argument( + "--enable_cpu_affinity", + default=False, + action="store_true", + help="Whether or not CPU affinity and balancing should be enabled. Currently only supported on NVIDIA hardware.", + ) + # Dynamo arguments + resource_args.add_argument( + "--dynamo_backend", + type=str, + choices=["no"] + [b.lower() for b in DYNAMO_BACKENDS], + help="Choose a backend to optimize your training with dynamo, see more at " + "https://github.com/pytorch/torchdynamo.", + ) + resource_args.add_argument( + "--dynamo_mode", + type=str, + default="default", + choices=TORCH_DYNAMO_MODES, + help="Choose a mode to optimize your training with dynamo.", + ) + resource_args.add_argument( + "--dynamo_use_fullgraph", + default=False, + action="store_true", + help="Whether to use full graph mode for dynamo or it is ok to break model into several subgraphs", + ) + resource_args.add_argument( + "--dynamo_use_dynamic", + default=False, + action="store_true", + help="Whether to enable dynamic shape tracing.", + ) + resource_args.add_argument( + "--dynamo_use_regional_compilation", + default=False, + action="store_true", + help="Whether to enable regional compilation.", + ) + + # Training Paradigm arguments + paradigm_args = parser.add_argument_group( + "Training Paradigm Arguments", "Arguments for selecting which training paradigm to be used." + ) + paradigm_args.add_argument( + "--use_deepspeed", + default=False, + action="store_true", + help="Whether to use deepspeed.", + ) + paradigm_args.add_argument( + "--use_fsdp", + default=False, + action="store_true", + help="Whether to use fsdp.", + ) + paradigm_args.add_argument( + "--use_parallelism_config", + default=False, + action="store_true", + help="Whether to use the parallelism config to configure the N-d distributed training.", + ) + paradigm_args.add_argument( + "--use_megatron_lm", + default=False, + action="store_true", + help="Whether to use Megatron-LM.", + ) + + # distributed GPU training arguments + distributed_args = parser.add_argument_group("Distributed GPUs", "Arguments related to distributed GPU training.") + distributed_args.add_argument( + "--gpu_ids", + default=None, + help="What GPUs (by id) should be used for training on this machine as a comma-separated list", + ) + distributed_args.add_argument( + "--same_network", + default=False, + action="store_true", + help="Whether all machines used for multinode training exist on the same local network.", + ) + distributed_args.add_argument( + "--machine_rank", type=int, default=None, help="The rank of the machine on which this script is launched." + ) + distributed_args.add_argument( + "--main_process_ip", type=str, default=None, help="The IP address of the machine of rank 0." + ) + distributed_args.add_argument( + "--main_process_port", + type=int, + default=None, + help="The port to use to communicate with the machine of rank 0.", + ) + distributed_args.add_argument( + "-t", + "--tee", + default="0", + type=str, + help="Tee std streams into a log file and also to console.", + ) + distributed_args.add_argument( + "--log_dir", + type=str, + default=None, + help=( + "Base directory to use for log files when using torchrun/torch.distributed.run as launcher. " + "Use with --tee to redirect std streams info log files." + ), + ) + distributed_args.add_argument( + "--role", + type=str, + default="default", + help="User-defined role for the workers.", + ) + # Rendezvous related arguments + distributed_args.add_argument( + "--rdzv_backend", + type=str, + default="static", + help="The rendezvous method to use, such as 'static' (the default) or 'c10d'", + ) + distributed_args.add_argument( + "--rdzv_conf", + type=str, + default="", + help="Additional rendezvous configuration (=,=,...).", + ) + distributed_args.add_argument( + "--max_restarts", + type=int, + default=0, + help="Maximum number of worker group restarts before failing.", + ) + distributed_args.add_argument( + "--monitor_interval", + type=float, + default=0.1, + help="Interval, in seconds, to monitor the state of workers.", + ) + parser.add_argument( + "-m", + "--module", + action="store_true", + help="Change each process to interpret the launch script as a Python module, executing with the same behavior as 'python -m'.", + ) + parser.add_argument( + "--no_python", + action="store_true", + help="Skip prepending the training script with 'python' - just execute it directly. Useful when the script is not a Python script.", + ) + + # TPU arguments + tpu_args = parser.add_argument_group("TPU", "Arguments related to TPU.") + tpu_args.add_argument( + "--tpu_cluster", + action="store_true", + dest="tpu_use_cluster", + help="Whether to use a GCP TPU pod for training.", + ) + tpu_args.add_argument( + "--no_tpu_cluster", + action="store_false", + dest="tpu_use_cluster", + help="Should not be passed explicitly, this is for internal use only.", + ) + tpu_args.add_argument( + "--tpu_use_sudo", + action="store_true", + help="Whether to use `sudo` when running the TPU training script in each pod.", + ) + tpu_args.add_argument( + "--vm", + type=str, + action="append", + help=( + "List of single Compute VM instance names. " + "If not provided we assume usage of instance groups. For TPU pods." + ), + ) + tpu_args.add_argument( + "--env", + type=str, + action="append", + help="List of environment variables to set on the Compute VM instances. For TPU pods.", + ) + tpu_args.add_argument( + "--main_training_function", + type=str, + default=None, + help="The name of the main function to be executed in your script (only for TPU training).", + ) + tpu_args.add_argument( + "--downcast_bf16", + action="store_true", + help="Whether when using bf16 precision on TPUs if both float and double tensors are cast to bfloat16 or if double tensors remain as float32.", + ) + + # DeepSpeed arguments + deepspeed_args = parser.add_argument_group("DeepSpeed Arguments", "Arguments related to DeepSpeed.") + deepspeed_args.add_argument( + "--deepspeed_config_file", + default=None, + type=str, + help="DeepSpeed config file.", + ) + deepspeed_args.add_argument( + "--zero_stage", + default=None, + type=int, + help="DeepSpeed's ZeRO optimization stage (useful only when `use_deepspeed` flag is passed). " + "If unspecified, will default to `2`.", + ) + deepspeed_args.add_argument( + "--offload_optimizer_device", + default=None, + type=str, + help="Decides where (none|cpu|nvme) to offload optimizer states (useful only when `use_deepspeed` flag is passed). " + "If unspecified, will default to 'none'.", + ) + deepspeed_args.add_argument( + "--offload_param_device", + default=None, + type=str, + help="Decides where (none|cpu|nvme) to offload parameters (useful only when `use_deepspeed` flag is passed). " + "If unspecified, will default to 'none'.", + ) + deepspeed_args.add_argument( + "--offload_optimizer_nvme_path", + default=None, + type=str, + help="Decides Nvme Path to offload optimizer states (useful only when `use_deepspeed` flag is passed). " + "If unspecified, will default to 'none'.", + ) + deepspeed_args.add_argument( + "--offload_param_nvme_path", + default=None, + type=str, + help="Decides Nvme Path to offload parameters (useful only when `use_deepspeed` flag is passed). " + "If unspecified, will default to 'none'.", + ) + deepspeed_args.add_argument( + "--gradient_accumulation_steps", + default=None, + type=int, + help="No of gradient_accumulation_steps used in your training script (useful only when `use_deepspeed` flag is passed). " + "If unspecified, will default to `1`.", + ) + deepspeed_args.add_argument( + "--gradient_clipping", + default=None, + type=float, + help="gradient clipping value used in your training script (useful only when `use_deepspeed` flag is passed). " + "If unspecified, will default to `1.0`.", + ) + deepspeed_args.add_argument( + "--zero3_init_flag", + default=None, + type=str, + help="Decides Whether (true|false) to enable `deepspeed.zero.Init` for constructing massive models. " + "Only applicable with DeepSpeed ZeRO Stage-3. If unspecified, will default to `true`.", + ) + deepspeed_args.add_argument( + "--zero3_save_16bit_model", + default=None, + type=str, + help="Decides Whether (true|false) to save 16-bit model weights when using ZeRO Stage-3. " + "Only applicable with DeepSpeed ZeRO Stage-3. If unspecified, will default to `false`.", + ) + deepspeed_args.add_argument( + "--deepspeed_hostfile", + default=None, + type=str, + help="DeepSpeed hostfile for configuring multi-node compute resources.", + ) + deepspeed_args.add_argument( + "--deepspeed_exclusion_filter", + default=None, + type=str, + help="DeepSpeed exclusion filter string when using multi-node setup.", + ) + deepspeed_args.add_argument( + "--deepspeed_inclusion_filter", + default=None, + type=str, + help="DeepSpeed inclusion filter string when using multi-node setup.", + ) + deepspeed_args.add_argument( + "--deepspeed_multinode_launcher", + default=None, + type=str, + help="DeepSpeed multi-node launcher to use, e.g. `pdsh`, `standard`, `openmpi`, `mvapich`, `mpich`, `slurm`, `nossh` (requires DeepSpeed >= 0.14.5). If unspecified, will default to `pdsh`.", + ) + deepspeed_args.add_argument( + "--deepspeed_moe_layer_cls_names", + default=None, + type=str, + help="comma-separated list of transformer MoE layer class names (case-sensitive) to wrap ,e.g, `MixtralSparseMoeBlock`, `Qwen2MoeSparseMoeBlock`, `JetMoEAttention,JetMoEBlock` ..." + " (useful only when `use_deepspeed` flag is passed).", + ) + + # fsdp arguments + fsdp_args = parser.add_argument_group("FSDP Arguments", "Arguments related to Fully Shared Data Parallelism.") + fsdp_args.add_argument( + "--fsdp_version", + type=str, + default="1", + choices=["1", "2"], + help="FSDP version to use. (useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_offload_params", + default="false", + type=str, + help="Decides Whether (true|false) to offload parameters and gradients to CPU. (useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_min_num_params", + type=int, + default=int(1e8), + help="FSDP's minimum number of parameters for Default Auto Wrapping. (useful only when `use_fsdp` flag is passed).", + ) + # We enable this for backwards compatibility, throw a warning if this is set in `FullyShardedDataParallelPlugin` + fsdp_args.add_argument( + "--fsdp_sharding_strategy", + type=str, + default="FULL_SHARD", + help="FSDP's sharding strategy. (useful only when `use_fsdp` flag is passed and `fsdp_version=1`).", + ) + fsdp_args.add_argument( + "--fsdp_reshard_after_forward", + type=str, + default="true", + help="FSDP's Reshard After Forward Strategy. (useful only when `use_fsdp` flag is passed). Supports either boolean (FSDP2) or `FULL_SHARD | SHARD_GRAD_OP | NO_RESHARD` (FSDP1).", + ) + fsdp_args.add_argument( + "--fsdp_auto_wrap_policy", + type=str, + default=None, + help="FSDP's auto wrap policy. (useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_transformer_layer_cls_to_wrap", + default=None, + type=str, + help="Transformer layer class name (case-sensitive) to wrap ,e.g, `BertLayer`, `GPTJBlock`, `T5Block` .... " + "(useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_backward_prefetch", + default=None, + type=str, + help="FSDP's backward prefetch policy. (useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_state_dict_type", + default=None, + type=str, + help="FSDP's state dict type. (useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_forward_prefetch", + default="false", + type=str, + help="If True, then FSDP explicitly prefetches the next upcoming " + "all-gather while executing in the forward pass (useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_use_orig_params", + default="true", + type=str, + help="If True, allows non-uniform `requires_grad` during init, which means support for interspersed frozen and trainable parameters." + " (useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_cpu_ram_efficient_loading", + default="true", + type=str, + help="If True, only the first process loads the pretrained model checkpoint while all other processes have empty weights. " + "Only applicable for 🤗 Transformers. When using this, `--fsdp_sync_module_states` needs to True. " + "(useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_sync_module_states", + default="true", + type=str, + help="If True, each individually wrapped FSDP unit will broadcast module parameters from rank 0." + " (useful only when `use_fsdp` flag is passed).", + ) + fsdp_args.add_argument( + "--fsdp_activation_checkpointing", + default="false", + type=str, + help="Decides Whether (true|false) intermediate activations are freed during the forward pass, and a checkpoint is left as a placeholder. (useful only when `use_fsdp` flag is passed).", + ) + + # megatron_lm args + megatron_lm_args = parser.add_argument_group("Megatron-LM Arguments", "Arguments related to Megatron-LM.") + megatron_lm_args.add_argument( + "--megatron_lm_tp_degree", + type=int, + default=1, + help="Megatron-LM's Tensor Parallelism (TP) degree. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_use_custom_fsdp", + type=bool, + default=False, + help="Whether to use custom FSDP. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_no_load_optim", + type=bool, + default=False, + help="Whether to not load optimizer. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_eod_mask_loss", + type=bool, + default=False, + help="Whether to use eod mask loss. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_overlap_cpu_optimizer_d2h_h2d", + type=bool, + default=False, + help="Whether to overlap CPU optimizer step, gradients D2H and updated parameters H2D. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_no_save_optim", + type=bool, + default=False, + help="Whether to not save optimizer. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_optimizer_cpu_offload", + type=bool, + default=False, + help="Whether to use CPU offload for optimizer. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_use_precision_aware_optimizer", + type=bool, + default=False, + help="Whether to use precision aware optimizer. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_decoder_last_pipeline_num_layers", + type=int, + default=None, + help="Megatron-LM's decoder last pipeline number of layers, default None is even split of transformer layers across all pipeline stages.", + ) + megatron_lm_args.add_argument( + "--megatron_lm_pp_degree", + type=int, + default=1, + help="Megatron-LM's Pipeline Parallelism (PP) degree. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_num_micro_batches", + type=int, + default=None, + help="Megatron-LM's number of micro batches when PP degree > 1. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_sequence_parallelism", + default=None, + type=str, + help="Decides Whether (true|false) to enable Sequence Parallelism when TP degree > 1. " + "(useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_recompute_activations", + default=None, + type=str, + help="Decides Whether (true|false) to enable Selective Activation Recomputation. " + "(useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_use_distributed_optimizer", + default=None, + type=str, + help="Decides Whether (true|false) to use distributed optimizer " + "which shards optimizer state and gradients across Data Pralellel (DP) ranks. " + "(useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_gradient_clipping", + default=1.0, + type=float, + help="Megatron-LM's gradient clipping value based on global L2 Norm (0 to disable). " + "(useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_recompute_granularity", + default=None, + type=str, + help="Megatron-LM's recompute granularity (full, selective). " + "(useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_recompute_method", + default=None, + type=str, + help="Megatron-LM's recompute method (uniform, block). (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_recompute_num_layers", + default=None, + type=int, + help="Megatron-LM's number of layers to recompute. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_attention_backend", + default=None, + type=str, + help="Decides Whether (true|false) to enable attention backend. " + "(useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_expert_model_parallel_size", + default=None, + type=int, + help="Megatron-LM's expert model parallel size. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_context_parallel_size", + default=None, + type=int, + help="Megatron-LM's context parallel size. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_attention_dropout", + default=None, + type=float, + help="Megatron-LM's attention dropout rate. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_hidden_dropout", + default=None, + type=float, + help="Megatron-LM's hidden dropout rate. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_attention_softmax_in_fp32", + default=None, + type=str, + help="Decides Whether (true|false) to use fp32 for attention softmax. " + "(useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_expert_tensor_parallel_size", + default=None, + type=int, + help="Megatron-LM's expert tensor parallel size. (useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_calculate_per_token_loss", + default=None, + type=str, + help="Decides Whether (true|false) to calculate per token loss. " + "(useful only when `use_megatron_lm` flag is passed).", + ) + megatron_lm_args.add_argument( + "--megatron_lm_use_rotary_position_embeddings", + default=None, + type=str, + help="Decides Whether (true|false) to use rotary position embeddings. " + "(useful only when `use_megatron_lm` flag is passed).", + ) + + # FP8 arguments + fp8_args = parser.add_argument_group( + "FP8 Arguments", "Arguments related to FP8 training (requires `--mixed_precision=fp8`)" + ) + fp8_args.add_argument( + "--fp8_backend", + type=str, + choices=["ao", "te", "msamp"], + help="Choose a backend to train with FP8 (ao: torchao, te: TransformerEngine, msamp: MS-AMP)", + ) + fp8_args.add_argument( + "--fp8_use_autocast_during_eval", + default=False, + action="store_true", + help="Whether to use FP8 autocast during eval mode (useful only when `--fp8_backend=te` is passed). Generally better metrics are found when this is not passed.", + ) + fp8_args.add_argument( + "--fp8_margin", + type=int, + default=0, + help="The margin to use for the gradient scaling (useful only when `--fp8_backend=te` is passed).", + ) + fp8_args.add_argument( + "--fp8_interval", + type=int, + default=1, + help="The interval to use for how often the scaling factor is recomputed (useful only when `--fp8_backend=te` is passed).", + ) + fp8_args.add_argument( + "--fp8_format", + type=str, + default="HYBRID", + choices=["HYBRID", "E4M3", "E5M2"], + help="The format to use for the FP8 recipe (useful only when `--fp8_backend=te` is passed).", + ) + fp8_args.add_argument( + "--fp8_amax_history_len", + type=int, + default=1024, + help="The length of the history to use for the scaling factor computation (useful only when `--fp8_backend=te` is passed).", + ) + fp8_args.add_argument( + "--fp8_amax_compute_algo", + type=str, + default="most_recent", + choices=["max", "most_recent"], + help="The algorithm to use for the scaling factor computation. (useful only when `--fp8_backend=te` is passed).", + ) + fp8_args.add_argument( + "--fp8_override_linear_precision", + type=lambda x: tuple(map(str_to_bool, x.split(","))), + default=(False, False, False), + help="Whether or not to execute `fprop`, `dgrad`, and `wgrad` GEMMS in higher precision. Should be passed in a comma-separated string of booleans (useful only when `--fp8_backend=te` is passed).", + ) + fp8_args.add_argument( + "--fp8_opt_level", + type=str, + default="O2", + choices=["O1", "O2"], + help="What level of 8-bit collective communication should be used with MS-AMP (useful only when `--fp8_backend=msamp` is passed).", + ) + fp8_args.add_argument( + "--fp8_enable_fsdp_float8_all_gather", + default="true", + type=str_to_bool, + help="Whether to enable FSDP2 float8 all gather (useful only when `--fp8_backend=ao` is passed).", + ) + fp8_args.add_argument( + "--fp8_pad_inner_dim", + default="true", + type=str_to_bool, + help="Whether to pad the inner dimension for FP8 GEMMs (useful only when `--fp8_backend=ao` is passed).", + ) + + # AWS arguments + aws_args = parser.add_argument_group("AWS Arguments", "Arguments related to AWS.") + aws_args.add_argument( + "--aws_access_key_id", + type=str, + default=None, + help="The AWS_ACCESS_KEY_ID used to launch the Amazon SageMaker training job", + ) + aws_args.add_argument( + "--aws_secret_access_key", + type=str, + default=None, + help="The AWS_SECRET_ACCESS_KEY used to launch the Amazon SageMaker training job.", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Whether to print out the torch.distributed stack trace when something fails.", + ) + parser.add_argument( + "training_script", + type=str, + help=( + "The full path to the script to be launched in parallel, followed by all the arguments for the training " + "script." + ), + ) + + # MPI arguments + mpirun_args = parser.add_argument_group("MPI Arguments", "Arguments related to mpirun for Multi-CPU") + mpirun_args.add_argument( + "--mpirun_hostfile", + type=str, + default=None, + help="Location for a hostfile for using Accelerate to launch a multi-CPU training job with mpirun. This will " + "get passed to the MPI --hostfile or -f parameter, depending on which MPI program is installed.", + ) + + # ParallelismConfig arguments + parallelism_config_args = parser.add_argument_group( + "ParallelismConfig Arguments", + "Arguments related to the ParallelismConfig used for distributed training.", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_dp_replicate_size", + type=int, + default=1, + help="The number of processes for data parallel training. Defaults to 1 (no data parallelism).", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_dp_shard_size", + type=int, + default=1, + help="The number of processes for FSDP sharding. Defaults to 1 (No FSDP sharding).", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_tp_size", + type=int, + default=1, + help="The number of processes for tensor parallel training. Defaults to 1 (no tensor parallelism).", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_cp_size", + type=int, + default=1, + help="The number of processese for context parallel training. Defaults to 1 (no context parallelism).", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_cp_backend", + type=str, + choices=["torch"], + default="torch", + help="Context Parallelism backend: torch (FSDP2) or deepspeed (ALST/Ulysses)", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_cp_comm_strategy", + type=str, + default="allgather", + help="The communication strategy for context parallel training. Defaults to 'allgather'. Other option is alltoall", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_sp_size", + type=int, + default=1, + help="The number of processese for context parallel training. Defaults to 1 (no context parallelism).", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_sp_backend", + type=str, + choices=["deepspeed"], + default="deepspeed", + help="Sequence Parallelism backend: deepspeed (ALST/Ulysses)", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_sp_seq_length", + type=str, + default=None, + help="Sequence length for when batches are all of the same length. For variable sequence lengths across batches set `parallelism_config_sp_seq_length_is_variable=True`", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_sp_seq_length_is_variable", + type=bool, + default=True, + help="If `True` will work with a sequence length that may change between batches, in which case `parallelism_config_sp_seq_length` value can be set to anything divisible by sp size or remain unset. If `False` then `parallelism_config_sp_seq_length` needs to match the batch's sequence length dimension. The default is `True`.", + ) + + parallelism_config_args.add_argument( + "--parallelism_config_sp_attn_implementation", + type=str, + default="sdpa", + help="Attention implementation to use. Can be one of 'flash_attention_2', 'flash_attention_3', 'sdpa', or a hub-hosted kernel (e.g. 'kernels-community/flash-attn2'). Defaults to `sdpa`.", + ) + + # Other arguments of the training scripts + parser.add_argument("training_script_args", nargs=argparse.REMAINDER, help="Arguments of the training script.") + + if subparsers is not None: + parser.set_defaults(func=launch_command) + return parser + + +def simple_launcher(args): + cmd, current_env = prepare_simple_launcher_cmd_env(args) + + process = subprocess.Popen(cmd, env=current_env) + process.wait() + if process.returncode != 0: + if not args.quiet: + raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) + else: + sys.exit(1) + + +def multi_gpu_launcher(args): + import torch.distributed.run as distrib_run + + current_env = prepare_multi_gpu_env(args) + if not check_cuda_p2p_ib_support(): + message = "Using RTX 4000 series which doesn't support faster communication speedups. Ensuring P2P and IB communications are disabled." + warn = False + if "NCCL_P2P_DISABLE" not in current_env: + current_env["NCCL_P2P_DISABLE"] = "1" + warn = True + if "NCCL_IB_DISABLE" not in current_env: + current_env["NCCL_IB_DISABLE"] = "1" + warn = True + if warn: + logger.warning(message) + + debug = getattr(args, "debug", False) + args = _filter_args( + args, + distrib_run.get_args_parser(), + ["--training_script", args.training_script, "--training_script_args", args.training_script_args], + ) + + with patch_environment(**current_env): + try: + distrib_run.run(args) + except Exception: + if is_rich_available() and debug: + console = get_console() + console.print("\n[bold red]Using --debug, `torch.distributed` Stack Trace:[/bold red]") + console.print_exception(suppress=[__file__], show_locals=False) + else: + raise + + +def deepspeed_launcher(args): + import torch.distributed.run as distrib_run + + if not is_deepspeed_available(): + raise ImportError("DeepSpeed is not installed => run `pip3 install deepspeed` or build it from source.") + else: + from deepspeed.launcher.runner import DEEPSPEED_ENVIRONMENT_NAME + + cmd, current_env = prepare_deepspeed_cmd_env(args) + if not check_cuda_p2p_ib_support(): + message = "Using RTX 4000 series which doesn't support faster communication speedups. Ensuring P2P and IB communications are disabled." + warn = False + if "NCCL_P2P_DISABLE" not in current_env: + current_env["NCCL_P2P_DISABLE"] = "1" + warn = True + if "NCCL_IB_DISABLE" not in current_env: + current_env["NCCL_IB_DISABLE"] = "1" + warn = True + if warn: + logger.warning(message) + + if args.num_machines > 1 and args.deepspeed_multinode_launcher != DEEPSPEED_MULTINODE_LAUNCHERS[1]: + with open(DEEPSPEED_ENVIRONMENT_NAME, "a") as f: + valid_env_items = convert_dict_to_env_variables(current_env) + if len(valid_env_items) > 1: + f.writelines(valid_env_items) + + process = subprocess.Popen(cmd, env=current_env) + process.wait() + if process.returncode != 0: + if not args.quiet: + raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) + else: + sys.exit(1) + else: + debug = getattr(args, "debug", False) + args = _filter_args( + args, + distrib_run.get_args_parser(), + ["--training_script", args.training_script, "--training_script_args", args.training_script_args], + ) + with patch_environment(**current_env): + try: + distrib_run.run(args) + except Exception: + if is_rich_available() and debug: + console = get_console() + console.print("\n[bold red]Using --debug, `torch.distributed` Stack Trace:[/bold red]") + console.print_exception(suppress=[__file__], show_locals=False) + else: + raise + + +def tpu_launcher(args): + import torch_xla.distributed.xla_multiprocessing as xmp + + if args.no_python: + raise ValueError("--no_python cannot be used with TPU launcher") + + args, current_env = prepare_tpu(args, {}) + + if args.module: + mod_name = args.training_script + else: + # Import training_script as a module + script_path = Path(args.training_script) + sys.path.append(str(script_path.parent.resolve())) + mod_name = script_path.stem + + mod = importlib.import_module(mod_name) + if not hasattr(mod, args.main_training_function): + raise ValueError( + f"Your training script should have a function named {args.main_training_function}, or you should pass a " + "different value to `--main_training_function`." + ) + + # Patch sys.argv + sys.argv = [mod.__file__] + args.training_script_args + + main_function = getattr(mod, args.main_training_function) + with patch_environment(**current_env): + xmp.spawn(PrepareForLaunch(main_function), args=()) + + +def tpu_pod_launcher(args): + from torch_xla.distributed import xla_dist + + current_env = {} + args, current_env = prepare_tpu(args, current_env, True) + debug = getattr(args, "debug", False) + + training_script = args.training_script + training_script_args = args.training_script_args + new_args = _filter_args( + args, xla_dist.get_args_parser(), ["--tpu", args.tpu_name, "--positional", "", "--restart-tpuvm-pod-server"] + ) + + if args.tpu_use_sudo: + new_cmd = ["sudo"] + else: + new_cmd = [] + + new_cmd += [ + "accelerate-launch", + "--tpu", + "--no_tpu_cluster", + "--num_machines", + "1", + "--mixed_precision", + "no", + "--dynamo_backend", + "no", + "--num_processes", + str(args.num_processes), + "--main_training_function", + str(args.main_training_function), + training_script, + ] + training_script_args + + new_args.positional = new_cmd + bad_flags = "" + for arg in vars(new_args): + if arg.startswith("docker_"): + value = getattr(new_args, arg) + if value != "" and value is not None: + bad_flags += f'{arg}="{value}"\n' + if bad_flags != "": + raise ValueError( + f"Docker containers are not supported for TPU pod launcher currently, please remove the following flags:\n{bad_flags}" + ) + new_args.env = [f"{k}={v}" for k, v in current_env.items()] + new_args.env.append("ACCELERATE_IN_TPU_POD=1") + try: + xla_dist.resolve_and_execute(new_args) + except Exception: + if is_rich_available() and debug: + console = get_console() + console.print("\n[bold red]Using --debug, `torch_xla.xla_dist` Stack Trace:[/bold red]") + console.print_exception(suppress=[__file__], show_locals=False) + else: + raise + + +def sagemaker_launcher(sagemaker_config: SageMakerConfig, args): + if not is_sagemaker_available(): + raise ImportError( + "Please install sagemaker to be able to launch training on Amazon SageMaker with `pip install accelerate[sagemaker]`" + ) + if args.module or args.no_python: + raise ValueError( + "SageMaker requires a python training script file and cannot be used with --module or --no_python" + ) + + from sagemaker.huggingface import HuggingFace + + args, sagemaker_inputs = prepare_sagemager_args_inputs(sagemaker_config, args) + + huggingface_estimator = HuggingFace(**args) + + huggingface_estimator.fit(inputs=sagemaker_inputs) + print(f"You can find your model data at: {huggingface_estimator.model_data}") + + +def _validate_launch_command(args): + # Sanity checks + if sum([args.multi_gpu, args.cpu, args.tpu, args.use_deepspeed, args.use_fsdp]) > 1: + raise ValueError( + "You can only use one of `--cpu`, `--multi_gpu`, `--tpu`, `--use_deepspeed`, `--use_fsdp` at a time." + ) + if args.multi_gpu and (args.num_processes is not None) and (args.num_processes < 2): + raise ValueError("You need to use at least 2 processes to use `--multi_gpu`.") + + if (not args.use_fsdp or args.fsdp_version == 1) and args.use_parallelism_config: + raise ValueError("You cannot use `--use_parallelism_config` without `--use_fsdp` and `--fsdp_version=2`. ") + + defaults = None + warned = [] + mp_from_config_flag = False + # Get the default from the config file. + if args.config_file is not None or os.path.isfile(default_config_file) and not args.cpu: + defaults = load_config_from_file(args.config_file) + if ( + not args.multi_gpu + and not args.tpu + and not args.tpu_use_cluster + and not args.use_deepspeed + and not args.use_fsdp + and not args.use_megatron_lm + ): + args.use_deepspeed = defaults.distributed_type == DistributedType.DEEPSPEED + args.multi_gpu = ( + True + if defaults.distributed_type + in ( + DistributedType.MULTI_GPU, + DistributedType.MULTI_NPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_XPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + ) + else False + ) + args.tpu = defaults.distributed_type == DistributedType.XLA + args.use_fsdp = defaults.distributed_type == DistributedType.FSDP + args.use_megatron_lm = defaults.distributed_type == DistributedType.MEGATRON_LM + args.tpu_use_cluster = defaults.tpu_use_cluster if args.tpu else False + args.use_parallelism_config = defaults.parallelism_config != {} + if args.gpu_ids is None: + if defaults.gpu_ids is not None: + args.gpu_ids = defaults.gpu_ids + else: + args.gpu_ids = "all" + + if args.multi_gpu and args.num_machines is None: + args.num_machines = defaults.num_machines + + if len(args.gpu_ids.split(",")) < 2 and (args.gpu_ids != "all") and args.multi_gpu and args.num_machines <= 1: + raise ValueError( + "Less than two GPU ids were configured and tried to run on on multiple GPUs. " + "Please ensure at least two are specified for `--gpu_ids`, or use `--gpu_ids='all'`." + ) + if defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE: + # Update args with the defaults + for name, attr in defaults.__dict__.items(): + if isinstance(attr, dict): + # Copy defaults.somedict.somearg to args.somearg and + # defaults.fsdp_config.x to args.fsdp_x + for key, value in attr.items(): + if name == "fsdp_config" and not key.startswith("fsdp"): + key = "fsdp_" + key + elif name == "fp8_config" and not key.startswith("fp8"): + key = "fp8_" + key + elif name == "dynamo_config" and not key.startswith("dynamo_"): + key = "dynamo_" + key + if hasattr(args, "nondefault") and key not in args.nondefault: + setattr(args, key, value) + elif ( + name not in ["compute_environment", "mixed_precision", "distributed_type"] + and getattr(args, name, None) is None + ): + # Those args are handled separately + setattr(args, name, attr) + if not args.debug: + args.debug = defaults.debug + + if not args.mixed_precision: + if defaults.mixed_precision is None: + args.mixed_precision = "no" + else: + args.mixed_precision = defaults.mixed_precision + mp_from_config_flag = True + else: + native_amp = is_bf16_available(True) + if ( + args.mixed_precision == "bf16" + and not native_amp + and not (args.tpu and is_torch_xla_available(check_is_tpu=True)) + ): + raise ValueError("bf16 mixed precision requires PyTorch >= 1.10 and a supported device.") + + # Silently set the default here + if args.dynamo_backend is None: + args.dynamo_backend = "no" + if args.num_processes == -1: + raise ValueError("You need to manually pass in `--num_processes` using this config yaml.") + else: + if args.num_processes is None: + if is_xpu_available(): + args.num_processes = torch.xpu.device_count() + elif is_mlu_available(): + args.num_processes = torch.mlu.device_count() + elif is_sdaa_available(): + args.num_processes = torch.sdaa.device_count() + elif is_musa_available(): + args.num_processes = torch.musa.device_count() + elif is_npu_available(): + args.num_processes = torch.npu.device_count() + elif is_hpu_available(): + args.num_processes = torch.hpu.device_count() + elif is_neuron_available(): + args.num_processes = torch.neuron.device_count() + else: + args.num_processes = torch.cuda.device_count() + warned.append(f"\t`--num_processes` was set to a value of `{args.num_processes}`") + if args.debug is None: + args.debug = False + if ( + not args.multi_gpu + and args.num_processes > 1 + and ( + (is_xpu_available() and torch.xpu.device_count() > 1) + or (is_npu_available() and torch.npu.device_count() > 1) + or (is_hpu_available() and torch.hpu.device_count() > 1) + or (is_mlu_available() and torch.mlu.device_count() > 1) + or (is_sdaa_available() and torch.sdaa.device_count() > 1) + or (is_musa_available() and torch.musa.device_count() > 1) + or (is_neuron_available() and torch.neuron.device_count() > 1) + or (torch.cuda.is_available() and torch.cuda.device_count() > 1) + ) + ): + warned.append( + "\t\tMore than one GPU was found, enabling multi-GPU training.\n" + "\t\tIf this was unintended please pass in `--num_processes=1`." + ) + args.multi_gpu = True + if args.num_machines is None: + warned.append("\t`--num_machines` was set to a value of `1`") + args.num_machines = 1 + if args.mixed_precision is None: + warned.append("\t`--mixed_precision` was set to a value of `'no'`") + args.mixed_precision = "no" + if not hasattr(args, "use_cpu"): + args.use_cpu = args.cpu + if args.dynamo_backend is None: + warned.append("\t`--dynamo_backend` was set to a value of `'no'`") + args.dynamo_backend = "no" + if args.debug: + logger.debug("Running script in debug mode, expect distributed operations to be slightly slower.") + + is_aws_env_disabled = defaults is None or ( + defaults is not None and defaults.compute_environment != ComputeEnvironment.AMAZON_SAGEMAKER + ) + if is_aws_env_disabled and args.num_cpu_threads_per_process is None: + args.num_cpu_threads_per_process = get_int_from_env(["OMP_NUM_THREADS"], 1) + if args.use_cpu and args.num_processes >= 1 and get_int_from_env(["OMP_NUM_THREADS"], 0) == 0: + local_size = get_int_from_env( + ["MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], + max(int(args.num_processes / args.num_machines), 1), + ) + import psutil + + threads_per_process = int(psutil.cpu_count(logical=False) / local_size) + if threads_per_process > 1: + args.num_cpu_threads_per_process = threads_per_process + warned.append( + f"\t`--num_cpu_threads_per_process` was set to `{args.num_cpu_threads_per_process}` to improve out-of-box performance when training on CPUs" + ) + + if any(warned): + message = "The following values were not passed to `accelerate launch` and had defaults used instead:\n" + message += "\n".join(warned) + message += ( + "\nTo avoid this warning pass in values for each of the problematic parameters or run `accelerate config`." + ) + logger.warning(message) + return args, defaults, mp_from_config_flag + + +def launch_command(args): + args, defaults, mp_from_config_flag = _validate_launch_command(args) + # Use the proper launcher + if args.use_deepspeed and not args.cpu: + args.deepspeed_fields_from_accelerate_config = list(defaults.deepspeed_config.keys()) if defaults else [] + if mp_from_config_flag: + args.deepspeed_fields_from_accelerate_config.append("mixed_precision") + args.deepspeed_fields_from_accelerate_config = ",".join(args.deepspeed_fields_from_accelerate_config) + deepspeed_launcher(args) + elif args.use_fsdp and not args.cpu: + multi_gpu_launcher(args) + elif args.use_megatron_lm and not args.cpu: + multi_gpu_launcher(args) + elif args.multi_gpu and not args.cpu: + multi_gpu_launcher(args) + elif args.tpu and not args.cpu: + if args.tpu_use_cluster: + tpu_pod_launcher(args) + else: + tpu_launcher(args) + elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER: + sagemaker_launcher(defaults, args) + else: + simple_launcher(args) + + +def main(): + parser = launch_command_parser() + args = parser.parse_args() + launch_command(args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/__init__.py b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c2c851cc0b192ab8207d3fa68d7409868c84354c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .selection_menu import BulletMenu diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50144940895d111c3d6cf6069c48697483d36a8d Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/cursor.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/cursor.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c9765b6b3dc499c96b993c417f0b4b608d31e7b Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/cursor.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/helpers.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/helpers.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f91182a8692df211f3f1a40027a0c4be0203248 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/helpers.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/input.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/input.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f27a713d39c2df7415be1224b810b1ea18f403fd Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/input.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/keymap.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/keymap.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f506955632233c9994172d11b5e71acd683e76f Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/keymap.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/selection_menu.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/selection_menu.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3881c7eab60d27f869514f4e75f5ec27c39fa578 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/commands/menu/__pycache__/selection_menu.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/cursor.py b/kbidu_env/Lib/site-packages/accelerate/commands/menu/cursor.py new file mode 100644 index 0000000000000000000000000000000000000000..c1f0bb7b68025ae4fe0c2c76c095eb36b4e64f2c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/menu/cursor.py @@ -0,0 +1,65 @@ +# Copyright 2022 The HuggingFace Team and Brian Chao. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A utility for showing and hiding the terminal cursor on Windows and Linux, based on https://github.com/bchao1/bullet +""" + +import os +import sys +from contextlib import contextmanager + + +# Windows only +if os.name == "nt": + import ctypes + import msvcrt # noqa + + class CursorInfo(ctypes.Structure): + # _fields is a specific attr expected by ctypes + _fields_ = [("size", ctypes.c_int), ("visible", ctypes.c_byte)] + + +def hide_cursor(): + if os.name == "nt": + ci = CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = False + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == "posix": + sys.stdout.write("\033[?25l") + sys.stdout.flush() + + +def show_cursor(): + if os.name == "nt": + ci = CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = True + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == "posix": + sys.stdout.write("\033[?25h") + sys.stdout.flush() + + +@contextmanager +def hide(): + "Context manager to hide the terminal cursor" + try: + hide_cursor() + yield + finally: + show_cursor() diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/helpers.py b/kbidu_env/Lib/site-packages/accelerate/commands/menu/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..de46f37ddcf4591167e3e01791391e4b1729034f --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/menu/helpers.py @@ -0,0 +1,59 @@ +# Copyright 2022 The HuggingFace Team and Brian Chao. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A variety of helper functions and constants when dealing with terminal menu choices, based on +https://github.com/bchao1/bullet +""" + +import enum +import shutil +import sys + + +TERMINAL_WIDTH, _ = shutil.get_terminal_size() + +CURSOR_TO_CHAR = {"UP": "A", "DOWN": "B", "RIGHT": "C", "LEFT": "D"} + + +class Direction(enum.Enum): + UP = 0 + DOWN = 1 + + +def forceWrite(content, end=""): + sys.stdout.write(str(content) + end) + sys.stdout.flush() + + +def writeColor(content, color, end=""): + forceWrite(f"\u001b[{color}m{content}\u001b[0m", end) + + +def reset_cursor(): + forceWrite("\r") + + +def move_cursor(num_lines: int, direction: str): + forceWrite(f"\033[{num_lines}{CURSOR_TO_CHAR[direction.upper()]}") + + +def clear_line(): + forceWrite(" " * TERMINAL_WIDTH) + reset_cursor() + + +def linebreak(): + reset_cursor() + forceWrite("-" * TERMINAL_WIDTH) diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/input.py b/kbidu_env/Lib/site-packages/accelerate/commands/menu/input.py new file mode 100644 index 0000000000000000000000000000000000000000..f1270eaece9d4243e7282dcb31166feeeb9bdfc1 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/menu/input.py @@ -0,0 +1,84 @@ +# Copyright 2022 The HuggingFace Team and Brian Chao. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file contains utilities for handling input from the user and registering specific keys to specific functions, +based on https://github.com/bchao1/bullet +""" + +from .keymap import KEYMAP, get_character + + +def mark(key: str): + """ + Mark the function with the key code so it can be handled in the register + """ + + def decorator(func): + handle = getattr(func, "handle_key", []) + handle += [key] + func.handle_key = handle + return func + + return decorator + + +def mark_multiple(*keys: list[str]): + """ + Mark the function with the key codes so it can be handled in the register + """ + + def decorator(func): + handle = getattr(func, "handle_key", []) + handle += keys + func.handle_key = handle + return func + + return decorator + + +class KeyHandler(type): + """ + Metaclass that adds the key handlers to the class + """ + + def __new__(cls, name, bases, attrs): + new_cls = super().__new__(cls, name, bases, attrs) + if not hasattr(new_cls, "key_handler"): + new_cls.key_handler = {} + new_cls.handle_input = KeyHandler.handle_input + + for value in attrs.values(): + handled_keys = getattr(value, "handle_key", []) + for key in handled_keys: + new_cls.key_handler[key] = value + return new_cls + + @staticmethod + def handle_input(cls): + "Finds and returns the selected character if it exists in the handler" + char = get_character() + if char != KEYMAP["undefined"]: + char = ord(char) + handler = cls.key_handler.get(char) + if handler: + cls.current_selection = char + return handler(cls) + else: + return None + + +def register(cls): + """Adds KeyHandler metaclass to the class""" + return KeyHandler(cls.__name__, cls.__bases__, cls.__dict__.copy()) diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/keymap.py b/kbidu_env/Lib/site-packages/accelerate/commands/menu/keymap.py new file mode 100644 index 0000000000000000000000000000000000000000..787db12860fe21c6786dda69c34fcccab114f2f8 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/menu/keymap.py @@ -0,0 +1,133 @@ +# Copyright 2022 The HuggingFace Team and Brian Chao. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utilities relating to parsing raw characters from the keyboard, based on https://github.com/bchao1/bullet +""" + +import os +import string +import sys + + +ARROW_KEY_FLAG = 1 << 8 + +KEYMAP = { + "tab": ord("\t"), + "newline": ord("\r"), + "esc": 27, + "up": 65 + ARROW_KEY_FLAG, + "down": 66 + ARROW_KEY_FLAG, + "right": 67 + ARROW_KEY_FLAG, + "left": 68 + ARROW_KEY_FLAG, + "mod_int": 91, + "undefined": sys.maxsize, + "interrupt": 3, + "insert": 50, + "delete": 51, + "pg_up": 53, + "pg_down": 54, +} + +KEYMAP["arrow_begin"] = KEYMAP["up"] +KEYMAP["arrow_end"] = KEYMAP["left"] + +if sys.platform == "win32": + WIN_CH_BUFFER = [] + WIN_KEYMAP = { + b"\xe0H": KEYMAP["up"] - ARROW_KEY_FLAG, + b"\x00H": KEYMAP["up"] - ARROW_KEY_FLAG, + b"\xe0P": KEYMAP["down"] - ARROW_KEY_FLAG, + b"\x00P": KEYMAP["down"] - ARROW_KEY_FLAG, + b"\xe0M": KEYMAP["right"] - ARROW_KEY_FLAG, + b"\x00M": KEYMAP["right"] - ARROW_KEY_FLAG, + b"\xe0K": KEYMAP["left"] - ARROW_KEY_FLAG, + b"\x00K": KEYMAP["left"] - ARROW_KEY_FLAG, + } + +for i in range(10): + KEYMAP[str(i)] = ord(str(i)) + + +def get_raw_chars(): + "Gets raw characters from inputs" + if os.name == "nt": + import msvcrt + + encoding = "mbcs" + # Flush the keyboard buffer + while msvcrt.kbhit(): + msvcrt.getch() + if len(WIN_CH_BUFFER) == 0: + # Read the keystroke + ch = msvcrt.getch() + + # If it is a prefix char, get second part + if ch in (b"\x00", b"\xe0"): + ch2 = ch + msvcrt.getch() + # Translate actual Win chars to bullet char types + try: + chx = chr(WIN_KEYMAP[ch2]) + WIN_CH_BUFFER.append(chr(KEYMAP["mod_int"])) + WIN_CH_BUFFER.append(chx) + if ord(chx) in ( + KEYMAP["insert"] - 1 << 9, + KEYMAP["delete"] - 1 << 9, + KEYMAP["pg_up"] - 1 << 9, + KEYMAP["pg_down"] - 1 << 9, + ): + WIN_CH_BUFFER.append(chr(126)) + ch = chr(KEYMAP["esc"]) + except KeyError: + ch = ch2[1] + else: + ch = ch.decode(encoding) + else: + ch = WIN_CH_BUFFER.pop(0) + elif os.name == "posix": + import termios + import tty + + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(fd) + ch = sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + return ch + + +def get_character(): + "Gets a character from the keyboard and returns the key code" + char = get_raw_chars() + if ord(char) in [KEYMAP["interrupt"], KEYMAP["newline"]]: + return char + + elif ord(char) == KEYMAP["esc"]: + combo = get_raw_chars() + if ord(combo) == KEYMAP["mod_int"]: + key = get_raw_chars() + if ord(key) >= KEYMAP["arrow_begin"] - ARROW_KEY_FLAG and ord(key) <= KEYMAP["arrow_end"] - ARROW_KEY_FLAG: + return chr(ord(key) + ARROW_KEY_FLAG) + else: + return KEYMAP["undefined"] + else: + return get_raw_chars() + + else: + if char in string.printable: + return char + else: + return KEYMAP["undefined"] diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/menu/selection_menu.py b/kbidu_env/Lib/site-packages/accelerate/commands/menu/selection_menu.py new file mode 100644 index 0000000000000000000000000000000000000000..ca66b2ed6782f819ffc69bd0bb3be5b7139a3863 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/menu/selection_menu.py @@ -0,0 +1,145 @@ +# Copyright 2022 The HuggingFace Team and Brian Chao. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Main driver for the selection menu, based on https://github.com/bchao1/bullet +""" + +import builtins +import sys +from typing import Optional + +from ...utils.imports import _is_package_available +from . import cursor, input +from .helpers import Direction, clear_line, forceWrite, linebreak, move_cursor, reset_cursor, writeColor +from .keymap import KEYMAP + + +in_colab = False +try: + in_colab = _is_package_available("google.colab") +except ModuleNotFoundError: + pass + + +@input.register +class BulletMenu: + """ + A CLI menu to select a choice from a list of choices using the keyboard. + """ + + def __init__(self, prompt: Optional[str] = None, choices: list = []): + self.position = 0 + self.choices = choices + self.prompt = prompt + if sys.platform == "win32": + self.arrow_char = "*" + else: + self.arrow_char = "➔ " + + def write_choice(self, index, end: str = ""): + if sys.platform != "win32": + writeColor(self.choices[index], 32, end) + else: + forceWrite(self.choices[index], end) + + def print_choice(self, index: int): + "Prints the choice at the given index" + if index == self.position: + forceWrite(f" {self.arrow_char} ") + self.write_choice(index) + else: + forceWrite(f" {self.choices[index]}") + reset_cursor() + + def move_direction(self, direction: Direction, num_spaces: int = 1): + "Should not be directly called, used to move a direction of either up or down" + old_position = self.position + if direction == Direction.DOWN: + if self.position + 1 >= len(self.choices): + return + self.position += num_spaces + else: + if self.position - 1 < 0: + return + self.position -= num_spaces + clear_line() + self.print_choice(old_position) + move_cursor(num_spaces, direction.name) + self.print_choice(self.position) + + @input.mark(KEYMAP["up"]) + def move_up(self): + self.move_direction(Direction.UP) + + @input.mark(KEYMAP["down"]) + def move_down(self): + self.move_direction(Direction.DOWN) + + @input.mark(KEYMAP["newline"]) + def select(self): + move_cursor(len(self.choices) - self.position, "DOWN") + return self.position + + @input.mark(KEYMAP["interrupt"]) + def interrupt(self): + move_cursor(len(self.choices) - self.position, "DOWN") + raise KeyboardInterrupt + + @input.mark_multiple(*[KEYMAP[str(number)] for number in range(10)]) + def select_row(self): + index = int(chr(self.current_selection)) + movement = index - self.position + if index == self.position: + return + if index < len(self.choices): + if self.position > index: + self.move_direction(Direction.UP, -movement) + elif self.position < index: + self.move_direction(Direction.DOWN, movement) + else: + return + else: + return + + def run(self, default_choice: int = 0): + "Start the menu and return the selected choice" + if self.prompt: + linebreak() + forceWrite(self.prompt, "\n") + if in_colab: + forceWrite("Please input a choice index (starting from 0), and press enter", "\n") + else: + forceWrite("Please select a choice using the arrow or number keys, and selecting with enter", "\n") + self.position = default_choice + for i in range(len(self.choices)): + self.print_choice(i) + forceWrite("\n") + move_cursor(len(self.choices) - self.position, "UP") + with cursor.hide(): + while True: + if in_colab: + try: + choice = int(builtins.input()) + except ValueError: + choice = default_choice + else: + choice = self.handle_input() + if choice is not None: + reset_cursor() + for _ in range(len(self.choices) + 1): + move_cursor(1, "UP") + clear_line() + self.write_choice(choice, "\n") + return choice diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/merge.py b/kbidu_env/Lib/site-packages/accelerate/commands/merge.py new file mode 100644 index 0000000000000000000000000000000000000000..475b53b5bbb71b959057126f8667d7f61eb9d0e1 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/merge.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from accelerate.commands.utils import CustomArgumentParser +from accelerate.utils import merge_fsdp_weights + + +description = """Utility to merge the weights from multiple FSDP checkpoints into a single combined checkpoint. Should be used if +`SHARDED_STATE_DICT` was used for the model. Weights will be saved to `{output_path}`. + +This is a CPU-bound process and requires enough RAM to load the entire model state dict.""" + + +def merge_command(args): + merge_fsdp_weights( + args.checkpoint_directory, args.output_path, not args.unsafe_serialization, args.remove_checkpoint_dir + ) + + +def merge_command_parser(subparsers=None): + if subparsers is not None: + parser = subparsers.add_parser("merge-weights", description=description) + else: + parser = CustomArgumentParser(description=description) + + parser.add_argument("checkpoint_directory", type=str, help="A directory containing sharded weights saved by FSDP.") + parser.add_argument( + "output_path", + type=str, + help="The path to save the merged weights. Defaults to the current directory. ", + ) + parser.add_argument( + "--unsafe_serialization", + action="store_true", + default=False, + help="Whether to save the merged weights as `.bin` rather than `.safetensors` (not recommended).", + ) + parser.add_argument( + "--remove_checkpoint_dir", + action="store_true", + help="Whether to remove the checkpoint directory after merging.", + default=False, + ) + + if subparsers is not None: + parser.set_defaults(func=merge_command) + return parser + + +def main(): + parser = merge_command_parser() + args = parser.parse_args() + merge_command(args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/test.py b/kbidu_env/Lib/site-packages/accelerate/commands/test.py new file mode 100644 index 0000000000000000000000000000000000000000..a0d2f7bcf14727aa13e3438f4cd6e6f140f5bb2f --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/test.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from accelerate.test_utils import execute_subprocess_async, path_in_accelerate_package + + +def test_command_parser(subparsers=None): + if subparsers is not None: + parser = subparsers.add_parser("test") + else: + parser = argparse.ArgumentParser("Accelerate test command") + + parser.add_argument( + "--config_file", + default=None, + help=( + "The path to use to store the config file. Will default to a file named default_config.yaml in the cache " + "location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have " + "such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed " + "with 'huggingface'." + ), + ) + + if subparsers is not None: + parser.set_defaults(func=test_command) + return parser + + +def test_command(args): + script_name = path_in_accelerate_package("test_utils", "scripts", "test_script.py") + + if args.config_file is None: + test_args = [script_name] + else: + test_args = f"--config_file={args.config_file} {script_name}".split() + + cmd = ["accelerate-launch"] + test_args + result = execute_subprocess_async(cmd) + if result.returncode == 0: + print("Test is a success! You are ready for your distributed training!") + + +def main(): + parser = test_command_parser() + args = parser.parse_args() + test_command(args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/to_fsdp2.py b/kbidu_env/Lib/site-packages/accelerate/commands/to_fsdp2.py new file mode 100644 index 0000000000000000000000000000000000000000..ba920c47b69d2c80345e2de9e2c5a862d643c6ba --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/to_fsdp2.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python + +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum +import logging +from pathlib import Path + +import yaml + +from accelerate.commands.utils import CustomArgumentParser + + +class ConversionStatus(enum.Enum): + NOT_YET_IMPLEMENTED = 0 + REMOVED = -1 + + +ARGUMENT_KEY_MAPPING = { + # New keys in FSDP2 + "fsdp_version": "fsdp_version", + "fsdp_reshard_after_forward": "fsdp_reshard_after_forward", + # https://github.com/pytorch/torchtitan/blob/main/docs/fsdp.md + # https://huggingface.co/docs/accelerate/en/usage_guides/fsdp + "fsdp_auto_wrap_policy": "fsdp_auto_wrap_policy", + "fsdp_backward_prefetch": ConversionStatus.REMOVED, + "fsdp_forward_prefetch": ConversionStatus.NOT_YET_IMPLEMENTED, + "fsdp_cpu_ram_efficient_loading": "fsdp_cpu_ram_efficient_loading", + "fsdp_offload_params": "fsdp_offload_params", + "fsdp_sharding_strategy": "fsdp_reshard_after_forward", + "fsdp_state_dict_type": "fsdp_state_dict_type", + "fsdp_sync_module_states": ConversionStatus.REMOVED, + "fsdp_transformer_layer_cls_to_wrap": "fsdp_transformer_layer_cls_to_wrap", + "fsdp_min_num_params": "fsdp_min_num_params", + "fsdp_use_orig_params": ConversionStatus.REMOVED, + "fsdp_activation_checkpointing": "fsdp_activation_checkpointing", +} + +ARGUMENT_VALUE_MAPPING = { + "fsdp_sharding_strategy": { + "FULL_SHARD": True, + "SHARD_GRAD_OP": False, + "HYBRID_SHARD": True, + "HYBRID_SHARD_ZERO2": False, + "NO_SHARD": False, + }, + "fsdp_reshard_after_forward": { # Needed to convert newly created configs using FSDP1 to FSDP2 + "FULL_SHARD": True, + "SHARD_GRAD_OP": False, + "HYBRID_SHARD": True, + "HYBRID_SHARD_ZERO2": False, + "NO_SHARD": False, + }, +} + +logger = logging.getLogger(__name__) + + +def _validate_to_fsdp2_args(args): + if not Path(args.config_file).exists(): + raise FileNotFoundError(f"Config file {args.config_file} not found") + + if not args.overwrite and args.output_file is None: + raise ValueError("If --overwrite is not set, --output_file must be provided") + + if not args.overwrite and Path(args.output_file).exists(): + raise FileExistsError(f"Output file {args.output_file} already exists and --overwrite is not set") + + +def convert_config_to_fsdp2(config: dict) -> dict: + fsdp_config = config.get("fsdp_config", {}) + + if not fsdp_config: + logger.info("No FSDP config found in the config file, skipping conversion...") + return config + + new_fsdp_config = {} + + if fsdp_config.get("fsdp_version", 1) == 2: + logger.warning("Config already specifies FSDP2, skipping conversion...") + logger.warning( + "If the config doesn't use new argument names, change `fsdp_version` to `1` and rerun the command." + ) + return config + + for key, value in fsdp_config.items(): + conversion_status = ARGUMENT_KEY_MAPPING.get(key, None) + # Key not in the mapping at all: carry it over unchanged. (ConversionStatus + # values must fall through to the REMOVED / NOT_YET_IMPLEMENTED handling + # below, otherwise those FSDP1-only keys would leak into the FSDP2 config.) + if conversion_status is None: + new_fsdp_config[key] = value + continue + + if conversion_status == ConversionStatus.REMOVED: + logger.warning(f"Argument {key} has been removed in FSDP2, skipping this key...") + continue + + if conversion_status == ConversionStatus.NOT_YET_IMPLEMENTED: + logger.warning(f"Argument {key} is not yet implemented in FSDP2, skipping this key...") + continue + + if conversion_status is None: + logger.warning(f"Argument {key} is not being converted, skipping this key...") + new_fsdp_config[key] = value + else: + if key in ARGUMENT_VALUE_MAPPING: + value = ARGUMENT_VALUE_MAPPING[key].get(value, value) + new_fsdp_config[ARGUMENT_KEY_MAPPING[key]] = value + + new_fsdp_config["fsdp_version"] = 2 + config["fsdp_config"] = new_fsdp_config + return config + + +def to_fsdp2_command_parser(subparsers=None): + description = "Convert an Accelerate config from FSDP1 to FSDP2" + + if subparsers is not None: + parser = subparsers.add_parser("to-fsdp2", description=description) + else: + parser = CustomArgumentParser(description=description) + + parser.add_argument("--config_file", type=str, help="The config file to convert to FSDP2", required=True) + parser.add_argument( + "--overwrite", + action="store_true", + help="Overwrite the config file if it exists", + default=False, + ) + parser.add_argument( + "--output_file", + type=str, + help="The path to the output file to write the converted config to. If not provided, the input file will be overwritten (if --overwrite is set)", + default=None, + ) + if subparsers is not None: + parser.set_defaults(func=to_fsdp2_command) + + return parser + + +def load_config(config_file: str) -> dict: + with open(config_file) as f: + config = yaml.safe_load(f) + if not config: + raise ValueError("Config file is empty") + + return config + + +def to_fsdp2_command(args): + _validate_to_fsdp2_args(args) + config = load_config(args.config_file) + + if args.overwrite and args.output_file is None: + args.output_file = args.config_file + + new_config = convert_config_to_fsdp2(config) + + with open(args.output_file, "w") as f: + yaml.dump(new_config, f) diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/tpu.py b/kbidu_env/Lib/site-packages/accelerate/commands/tpu.py new file mode 100644 index 0000000000000000000000000000000000000000..fc0f07bf8697bfdb6484d3bf817f2e18b1313b00 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/tpu.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python + +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import subprocess + +from packaging.version import Version, parse + +from accelerate.commands.config.config_args import default_config_file, load_config_from_file + + +_description = "Run commands across TPU VMs for initial setup before running `accelerate launch`." + + +def tpu_command_parser(subparsers=None): + if subparsers is not None: + parser = subparsers.add_parser("tpu-config", description=_description) + else: + parser = argparse.ArgumentParser("Accelerate tpu-config command", description=_description) + # Core arguments + config_args = parser.add_argument_group( + "Config Arguments", "Arguments that can be configured through `accelerate config`." + ) + config_args.add_argument( + "--config_file", + type=str, + default=None, + help="Path to the config file to use for accelerate.", + ) + config_args.add_argument( + "--tpu_name", + default=None, + help="The name of the TPU to use. If not specified, will use the TPU specified in the config file.", + ) + config_args.add_argument( + "--tpu_zone", + default=None, + help="The zone of the TPU to use. If not specified, will use the zone specified in the config file.", + ) + pod_args = parser.add_argument_group("TPU Arguments", "Arguments for options ran inside the TPU.") + pod_args.add_argument( + "--use_alpha", + action="store_true", + help="Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.", + ) + pod_args.add_argument( + "--command_file", + default=None, + help="The path to the file containing the commands to run on the pod on startup.", + ) + pod_args.add_argument( + "--command", + action="append", + nargs="+", + help="A command to run on the pod. Can be passed multiple times.", + ) + pod_args.add_argument( + "--install_accelerate", + action="store_true", + help="Whether to install accelerate on the pod. Defaults to False.", + ) + pod_args.add_argument( + "--accelerate_version", + default="latest", + help="The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify 'dev' to install from GitHub.", + ) + pod_args.add_argument( + "--debug", action="store_true", help="If set, will print the command that would be run instead of running it." + ) + + if subparsers is not None: + parser.set_defaults(func=tpu_command_launcher) + return parser + + +def tpu_command_launcher(args): + defaults = None + + # Get the default from the config file if it exists. + if args.config_file is not None or os.path.isfile(default_config_file): + defaults = load_config_from_file(args.config_file) + if not args.command_file and defaults.command_file is not None and not args.command: + args.command_file = defaults.command_file + if not args.command and defaults.commands is not None: + args.command = defaults.commands + if not args.tpu_name: + args.tpu_name = defaults.tpu_name + if not args.tpu_zone: + args.tpu_zone = defaults.tpu_zone + if args.accelerate_version == "dev": + args.accelerate_version = "git+https://github.com/huggingface/accelerate.git" + elif args.accelerate_version == "latest": + args.accelerate_version = "accelerate -U" + elif isinstance(parse(args.accelerate_version), Version): + args.accelerate_version = f"accelerate=={args.accelerate_version}" + + if not args.command_file and not args.command: + raise ValueError("You must specify either a command file or a command to run on the pod.") + + if args.command_file: + with open(args.command_file) as f: + args.command = [f.read().splitlines()] + + # To turn list of lists into list of strings + if isinstance(args.command[0], list): + args.command = [line for cmd in args.command for line in cmd] + # Default to the shared folder and install accelerate + new_cmd = ["cd /usr/share"] + if args.install_accelerate: + new_cmd += [f"pip install {args.accelerate_version}"] + new_cmd += args.command + args.command = "; ".join(new_cmd) + + # Then send it to gcloud + # Eventually try to use google-api-core to do this instead of subprocess + cmd = ["gcloud"] + if args.use_alpha: + cmd += ["alpha"] + cmd += [ + "compute", + "tpus", + "tpu-vm", + "ssh", + args.tpu_name, + "--zone", + args.tpu_zone, + "--command", + args.command, + "--worker", + "all", + ] + if args.debug: + print(f"Running {' '.join(cmd)}") + return + subprocess.run(cmd) + print("Successfully setup pod.") + + +def main(): + parser = tpu_command_parser() + args = parser.parse_args() + + tpu_command_launcher(args) diff --git a/kbidu_env/Lib/site-packages/accelerate/commands/utils.py b/kbidu_env/Lib/site-packages/accelerate/commands/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..326f37d7f93de2417e4171e5ffe91193fb97225c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/commands/utils.py @@ -0,0 +1,123 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + + +class _StoreAction(argparse.Action): + """ + Custom action that allows for `-` or `_` to be passed in for an argument. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + new_option_strings = [] + for option_string in self.option_strings: + new_option_strings.append(option_string) + if "_" in option_string[2:]: + # Add `-` version to the option string + new_option_strings.append(option_string.replace("_", "-")) + self.option_strings = new_option_strings + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + if not hasattr(namespace, "nondefault"): + namespace.nondefault = set() + namespace.nondefault.add(self.dest) + + +class _StoreConstAction(_StoreAction): + """ + Same as `argparse._StoreConstAction` but uses the custom `_StoreAction`. + """ + + def __init__(self, option_strings, dest, const, default=None, required=False, help=None): + super().__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help, + ) + + def __call__(self, parser, namespace, values, option_string=None): + super().__call__(parser, namespace, self.const, option_string) + + +class _StoreTrueAction(_StoreConstAction): + """ + Same as `argparse._StoreTrueAction` but uses the custom `_StoreConstAction`. + """ + + def __init__( + self, + option_strings, + dest, + default=None, + required=False, + help=None, + ): + super().__init__( + option_strings=option_strings, dest=dest, const=True, default=default, required=required, help=help + ) + + +class CustomArgumentGroup(argparse._ArgumentGroup): + """ + Custom argument group that allows for the use of `-` or `_` in arguments passed and overrides the help for each + when applicable. + """ + + def _add_action(self, action): + args = vars(action) + if isinstance(action, argparse._StoreTrueAction): + action = _StoreTrueAction( + args["option_strings"], args["dest"], args["default"], args["required"], args["help"] + ) + elif isinstance(action, argparse._StoreConstAction): + action = _StoreConstAction( + args["option_strings"], + args["dest"], + args["const"], + args["default"], + args["required"], + args["help"], + ) + elif isinstance(action, argparse._StoreAction): + action = _StoreAction(**args) + action = super()._add_action(action) + return action + + +class CustomArgumentParser(argparse.ArgumentParser): + """ + Custom argument parser that allows for the use of `-` or `_` in arguments passed and overrides the help for each + when applicable. + """ + + def add_argument(self, *args, **kwargs): + if "action" in kwargs: + # Translate action -> class + if kwargs["action"] == "store_true": + kwargs["action"] = _StoreTrueAction + else: + kwargs["action"] = _StoreAction + super().add_argument(*args, **kwargs) + + def add_argument_group(self, *args, **kwargs): + group = CustomArgumentGroup(self, *args, **kwargs) + self._action_groups.append(group) + return group diff --git a/kbidu_env/Lib/site-packages/accelerate/data_loader.py b/kbidu_env/Lib/site-packages/accelerate/data_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..f370ea3fb79b974c7790f6708f77716a406d37de --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/data_loader.py @@ -0,0 +1,1469 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import math +from contextlib import suppress +from typing import Callable, Optional, Union + +import torch +from packaging import version +from torch.utils.data import BatchSampler, DataLoader, IterableDataset, RandomSampler + +from .logging import get_logger +from .state import DistributedType, GradientState, PartialState, is_torch_xla_available +from .utils import ( + RNGType, + broadcast, + broadcast_object_list, + compare_versions, + concatenate, + find_batch_size, + get_data_structure, + initialize_tensors, + is_datasets_available, + is_torch_version, + is_torchdata_stateful_dataloader_available, + send_to_device, + slice_tensors, + synchronize_rng_states, +) + + +logger = get_logger(__name__) + +# kwargs of the DataLoader in min version 2.0 +_PYTORCH_DATALOADER_KWARGS = { + "batch_size": 1, + "shuffle": False, + "sampler": None, + "batch_sampler": None, + "num_workers": 0, + "collate_fn": None, + "pin_memory": False, + "drop_last": False, + "timeout": 0, + "worker_init_fn": None, + "multiprocessing_context": None, + "generator": None, + "prefetch_factor": 2, + "persistent_workers": False, + "pin_memory_device": "", +} + +# kwargs added after by version +_PYTORCH_DATALOADER_ADDITIONAL_KWARGS = {"2.6.0": {"in_order": True}} + +for v, additional_kwargs in _PYTORCH_DATALOADER_ADDITIONAL_KWARGS.items(): + if is_torch_version(">=", v): + _PYTORCH_DATALOADER_KWARGS.update(additional_kwargs) + + +class SeedableRandomSampler(RandomSampler): + """ + Same as a random sampler, except that in `__iter__` a seed can be used. + + Needed specifically in distributed cases, when the random generator for each GPU needs to start from the same seed + and be fully reproducible on multiple iterations. + + If a custom `generator` is passed, it will rely on its initial seed as well as the current iteration it is on + (stored in `self.epoch`). + """ + + def __init__(self, *args, **kwargs): + data_seed = kwargs.pop("data_seed", None) + super().__init__(*args, **kwargs) + + self.initial_seed = data_seed if data_seed is not None else torch.random.initial_seed() + self.epoch = 0 + + def __iter__(self): + if self.generator is None: + self.generator = torch.Generator( + device=torch.get_default_device() if hasattr(torch, "get_default_device") else "cpu" + ) + self.generator.manual_seed(self.initial_seed) + + # Allow `self.epoch` to modify the seed of the generator + seed = self.epoch + self.initial_seed + # print("Setting seed at epoch", self.epoch, seed) + self.generator.manual_seed(seed) + yield from super().__iter__() + self.set_epoch(self.epoch + 1) + + def set_epoch(self, epoch: int): + "Sets the current iteration of the sampler." + self.epoch = epoch + + +class BatchSamplerShard(BatchSampler): + """ + Wraps a PyTorch `BatchSampler` to generate batches for one of the processes only. Instances of this class will + always yield a number of batches that is a round multiple of `num_processes` and that all have the same size. + Depending on the value of the `drop_last` attribute of the batch sampler passed, it will either stop the iteration + at the first batch that would be too small / not present on all processes or loop with indices from the beginning. + + Args: + batch_sampler (`torch.utils.data.sampler.BatchSampler`): + The batch sampler to split in several shards. + num_processes (`int`, *optional*, defaults to 1): + The number of processes running concurrently. + process_index (`int`, *optional*, defaults to 0): + The index of the current process. + split_batches (`bool`, *optional*, defaults to `False`): + Whether the shards should be created by splitting a batch to give a piece of it on each process, or by + yielding different full batches on each process. + + On two processes with a sampler of `[[0, 1, 2, 3], [4, 5, 6, 7]]`, this will result in: + + - the sampler on process 0 to yield `[0, 1, 2, 3]` and the sampler on process 1 to yield `[4, 5, 6, 7]` if + this argument is set to `False`. + - the sampler on process 0 to yield `[0, 1]` then `[4, 5]` and the sampler on process 1 to yield `[2, 3]` + then `[6, 7]` if this argument is set to `True`. + even_batches (`bool`, *optional*, defaults to `True`): + Whether or not to loop back at the beginning of the sampler when the number of samples is not a round + multiple of (original batch size / number of processes). + + + + `BatchSampler`s with varying batch sizes are not enabled by default. To enable this behaviour, set `even_batches` + equal to `False` + + """ + + def __init__( + self, + batch_sampler: BatchSampler, + num_processes: int = 1, + process_index: int = 0, + split_batches: bool = False, + even_batches: bool = True, + ): + self.batch_sampler = batch_sampler + self.num_processes = num_processes + self.process_index = process_index + self.split_batches = split_batches + self.even_batches = even_batches + self.batch_size = getattr(batch_sampler, "batch_size", None) + self.drop_last = getattr(batch_sampler, "drop_last", False) + if split_batches and (self.batch_size is None or self.batch_size % num_processes != 0): + raise ValueError( + f"To use `BatchSamplerShard` in `split_batches` mode, the batch size ({self.batch_size}) " + f"needs to be a round multiple of the number of processes ({num_processes})." + ) + + @property + def total_length(self): + return len(self.batch_sampler) + + def __len__(self): + if self.split_batches: + # Split batches does not change the length of the batch sampler + return len(self.batch_sampler) + if len(self.batch_sampler) % self.num_processes == 0: + # If the length is a round multiple of the number of processes, it's easy. + return len(self.batch_sampler) // self.num_processes + length = len(self.batch_sampler) // self.num_processes + if self.drop_last: + # Same if we drop the remainder. + return length + elif self.even_batches: + # When we even batches we always get +1 + return length + 1 + else: + # Otherwise it depends on the process index. + return length + 1 if self.process_index < len(self.batch_sampler) % self.num_processes else length + + def __iter__(self): + return self._iter_with_split() if self.split_batches else self._iter_with_no_split() + + def _iter_with_split(self): + initial_data = [] + batch_length = self.batch_sampler.batch_size // self.num_processes + for idx, batch in enumerate(self.batch_sampler): + if idx == 0: + initial_data = batch + if len(batch) == self.batch_size: + # If the batch is full, we yield the part of it this process is responsible of. + yield batch[batch_length * self.process_index : batch_length * (self.process_index + 1)] + + # If drop_last is True of the last batch was full, iteration is over, otherwise... + if not self.drop_last and len(initial_data) > 0 and len(batch) < self.batch_size: + if not self.even_batches: + if len(batch) > batch_length * self.process_index: + yield batch[batch_length * self.process_index : batch_length * (self.process_index + 1)] + else: + # For degenerate cases where the dataset has less than num_process * batch_size samples + while len(initial_data) < self.batch_size: + initial_data += initial_data + batch = batch + initial_data + yield batch[batch_length * self.process_index : batch_length * (self.process_index + 1)] + + def _iter_with_no_split(self): + initial_data = [] + batch_to_yield = None + for idx, batch in enumerate(self.batch_sampler): + # We gather the initial indices in case we need to circle back at the end. + if not self.drop_last and idx < self.num_processes: + if self.batch_size is None: + # If batch size is None, `batch` is considered to be a list of indices with dynamic length. + initial_data.append(batch) + else: + initial_data += batch + # We identify the batch to yield but wait until we ar sure every process gets a full batch before actually + # yielding it. + if idx % self.num_processes == self.process_index: + batch_to_yield = batch + if idx % self.num_processes == self.num_processes - 1 and ( + self.batch_size is None or len(batch) == self.batch_size + ): + yield batch_to_yield + batch_to_yield = None + + # If drop_last is True, iteration is over, otherwise... + if not self.drop_last and len(initial_data) > 0: + if not self.even_batches: + if batch_to_yield: + yield batch_to_yield + else: + # ... we yield the complete batch we had saved before if it has the proper length + if batch_to_yield and (self.batch_size is None or len(batch_to_yield) == self.batch_size): + yield batch_to_yield + + # For degenerate cases where the dataset has less than num_process * batch_size samples + _min_length_needed = ( + self.num_processes * self.batch_size if self.batch_size is not None else self.num_processes + ) + while len(initial_data) < _min_length_needed: + initial_data += initial_data + + # If the last batch seen was of the proper size, it has been yielded by its process so we move to the next + if self.batch_size is None or len(batch) == self.batch_size: + batch = [] + idx += 1 + + # Make sure we yield a multiple of self.num_processes batches + cycle_index = 0 + while idx % self.num_processes != 0 or len(batch) > 0: + if self.batch_size is None: + batch = initial_data[cycle_index] + if idx % self.num_processes == self.process_index: + yield batch + cycle_index += 1 + else: + end_index = cycle_index + self.batch_size - len(batch) + batch += initial_data[cycle_index:end_index] + if idx % self.num_processes == self.process_index: + yield batch + cycle_index = end_index + batch = [] + idx += 1 + + +class IterableDatasetShard(IterableDataset): + """ + Wraps a PyTorch `IterableDataset` to generate samples for one of the processes only. Instances of this class will + always yield a number of samples that is a round multiple of the actual batch size (depending of the value of + `split_batches`, this is either `batch_size` or `batch_size x num_processes`). Depending on the value of the + `drop_last` attribute of the batch sampler passed, it will either stop the iteration at the first batch that would + be too small or loop with indices from the beginning. + + Args: + dataset (`torch.utils.data.dataset.IterableDataset`): + The batch sampler to split in several shards. + batch_size (`int`, *optional*, defaults to 1): + The size of the batches per shard (if `split_batches=False`) or the size of the batches (if + `split_batches=True`). + drop_last (`bool`, *optional*, defaults to `False`): + Whether or not to drop the last incomplete batch or complete the last batches by using the samples from the + beginning. + num_processes (`int`, *optional*, defaults to 1): + The number of processes running concurrently. + process_index (`int`, *optional*, defaults to 0): + The index of the current process. + split_batches (`bool`, *optional*, defaults to `False`): + Whether the shards should be created by splitting a batch to give a piece of it on each process, or by + yielding different full batches on each process. + + On two processes with an iterable dataset yielding of `[0, 1, 2, 3, 4, 5, 6, 7]`, this will result in: + + - the shard on process 0 to yield `[0, 1, 2, 3]` and the shard on process 1 to yield `[4, 5, 6, 7]` if this + argument is set to `False`. + - the shard on process 0 to yield `[0, 1, 4, 5]` and the sampler on process 1 to yield `[2, 3, 6, 7]` if + this argument is set to `True`. + """ + + def __init__( + self, + dataset: IterableDataset, + batch_size: int = 1, + drop_last: bool = False, + num_processes: int = 1, + process_index: int = 0, + split_batches: bool = False, + ): + if split_batches and batch_size > 1 and batch_size % num_processes != 0: + raise ValueError( + f"To use `IterableDatasetShard` in `split_batches` mode, the batch size ({batch_size}) " + f"needs to be a round multiple of the number of processes ({num_processes})." + ) + self.dataset: IterableDataset = dataset + self.batch_size = batch_size + self.drop_last = drop_last + self.num_processes = num_processes + self.process_index = process_index + self.split_batches = split_batches + + def set_epoch(self, epoch): + self.epoch = epoch + if hasattr(self.dataset, "set_epoch"): + self.dataset.set_epoch(epoch) + + def __len__(self): + # We will just raise the downstream error if the underlying dataset is not sized + if self.drop_last: + return (len(self.dataset) // (self.batch_size * self.num_processes)) * self.batch_size + else: + return math.ceil(len(self.dataset) / (self.batch_size * self.num_processes)) * self.batch_size + + def __iter__(self): + if ( + not hasattr(self.dataset, "set_epoch") + and hasattr(self.dataset, "generator") + and isinstance(self.dataset.generator, torch.Generator) + ): + self.dataset.generator.manual_seed(self.epoch) + real_batch_size = self.batch_size if self.split_batches else (self.batch_size * self.num_processes) + process_batch_size = (self.batch_size // self.num_processes) if self.split_batches else self.batch_size + process_slice = range(self.process_index * process_batch_size, (self.process_index + 1) * process_batch_size) + + first_batch = None + current_batch = [] + for element in self.dataset: + current_batch.append(element) + # Wait to have a full batch before yielding elements. + if len(current_batch) == real_batch_size: + for i in process_slice: + yield current_batch[i] + if first_batch is None: + first_batch = current_batch.copy() + current_batch = [] + + # Finished if drop_last is True, otherwise complete the last batch with elements from the beginning. + if not self.drop_last and len(current_batch) > 0: + if first_batch is None: + first_batch = current_batch.copy() + while len(current_batch) < real_batch_size: + current_batch += first_batch + for i in process_slice: + yield current_batch[i] + + +class DataLoaderStateMixin: + """ + Mixin class that adds a state to a `DataLoader` to keep track of the status inside the dataloader such as at the + end of the iteration, the number of items in the dataset in the last batch relative to the batch size, and other + useful information that might be needed. + + **Available attributes:** + + - **end_of_dataloader** (`bool`) -- Whether at the last iteration or batch + - **remainder** (`int`) -- The number of items that are remaining in the last batch, relative to the total + batch size + + + + Inheriters of this class should ensure that the class creates a `GradientState()` instance, stored in + `self.gradient_state`. + + + + """ + + def __init_subclass__(cls, **kwargs): + cls.end_of_dataloader = False + cls.remainder = -1 + + def reset(self): + self.end_of_dataloader = False + self.remainder = -1 + + def begin(self): + "Prepares the gradient state for the current dataloader" + self.reset() + with suppress(Exception): + if not self._drop_last: + length = getattr(self.dataset, "total_dataset_length", len(self.dataset)) + self.remainder = length % self.total_batch_size + self.gradient_state._add_dataloader(self) + + def end(self): + "Cleans up the gradient state after exiting the dataloader" + self.gradient_state._remove_dataloader(self) + + +class DataLoaderAdapter: + """ + A class which wraps around a PyTorch `DataLoader` (or variants of it) to be used with the `Accelerator`. For + compatibility reasons, this class inherits from the class it wraps around, so it can be used as a drop-in. + """ + + def __init__(self, dataset, use_stateful_dataloader=False, batch_sampler=None, **kwargs): + self.use_stateful_dataloader = use_stateful_dataloader + if is_torchdata_stateful_dataloader_available(): + from torchdata.stateful_dataloader import StatefulDataLoader + + if use_stateful_dataloader and not is_torchdata_stateful_dataloader_available(): + raise ImportError( + "StatefulDataLoader is not available. Please install torchdata version 0.8.0 or higher to use it." + ) + if use_stateful_dataloader: + torchdata_version = version.parse(importlib.metadata.version("torchdata")) + if ( + "in_order" in kwargs + and compare_versions(torchdata_version, "<", "0.11") + and is_torch_version(">=", "2.6.0") + ): + kwargs.pop("in_order") + self.base_dataloader = StatefulDataLoader(dataset, batch_sampler=batch_sampler, **kwargs) + else: + self.base_dataloader = DataLoader(dataset, batch_sampler=batch_sampler, **kwargs) + + if hasattr(self.base_dataloader, "state_dict"): + self.dl_state_dict = self.base_dataloader.state_dict() + + def __getattr__(self, name): + # Avoid infinite recursion if we try to access a nonexistent base_dataloader attribute. + if name == "base_dataloader": + raise AttributeError() + # Delegate attribute access to the internal dataloader + return getattr(self.base_dataloader, name) + + def state_dict(self): + return self.dl_state_dict + + def load_state_dict(self, state_dict): + self.base_dataloader.load_state_dict(state_dict) + + @property + def __class__(self): + """ + In order to maintain backwards compatibility with other code, we need to ensure `isinstance(obj, DataLoader)` + returns true. This is because some downstream code assumes that the `DataLoader` is the base class of the + object. + """ + return self.base_dataloader.__class__ + + def __len__(self): + return len(self.base_dataloader) + + def adjust_state_dict_for_prefetch(self): + """ + Adjusts the state dict for prefetching. Natively, this will adjust all of the iters yielded keys in + `self.dl_state_dict` by a factor of `num_processes - 1`, however if a custom correction is needed, this can be + overridden. + + This should modify `self.dl_state_dict` directly + """ + # The state dict will be off by a factor of `n-1` batch too many during DDP, + # so we need to adjust it here + if PartialState().distributed_type != DistributedType.NO: + factor = PartialState().num_processes - 1 + # When num_workers > 0, StatefulDataLoader uses _MultiProcessingDataLoaderIter + # which may not have _sampler_iter_yielded or _num_yielded in its state_dict + if "_sampler_iter_yielded" in self.dl_state_dict and self.dl_state_dict["_sampler_iter_yielded"] > 0: + self.dl_state_dict["_sampler_iter_yielded"] -= factor + if "_num_yielded" in self.dl_state_dict and self.dl_state_dict["_num_yielded"] > 0: + self.dl_state_dict["_num_yielded"] -= factor + if self.dl_state_dict.get("_index_sampler_state") is not None: + if ( + "samples_yielded" in self.dl_state_dict["_index_sampler_state"] + and self.dl_state_dict["_index_sampler_state"]["samples_yielded"] > 0 + ): + self.dl_state_dict["_index_sampler_state"]["samples_yielded"] -= self.batch_size * factor + + def _update_state_dict(self): + # The state_dict of the underlying base_dataloader may be ahead of what is currently being yielded. + # E.g. the implementation of DataLoaderShard involves having an underlying iterator 1 element ahead of + # what it wants to yield. + # + # _update_state_dict is called to snapshot the state_dict that would properly recover the DataLoaderAdapter. + if hasattr(self.base_dataloader, "state_dict"): + self.dl_state_dict = self.base_dataloader.state_dict() + # Potentially modify the state_dict to adjust for prefetching + self.adjust_state_dict_for_prefetch() + # Then tag if we are at the end of the dataloader + self.dl_state_dict["_iterator_finished"] = self.end_of_dataloader + + +class DataLoaderShard(DataLoaderAdapter, DataLoaderStateMixin): + """ + Subclass of `DataLoaderAdapter` that will deal with device placement and current distributed setup. + + Args: + dataset (`torch.utils.data.dataset.Dataset`): + The dataset to use to build this dataloader. + device (`torch.device`, *optional*): + If passed, the device to put all batches on. + rng_types (list of `str` or [`~utils.RNGType`]): + The list of random number generators to synchronize at the beginning of each iteration. Should be one or + several of: + + - `"torch"`: the base torch random number generator + - `"cuda"`: the CUDA random number generator (GPU only) + - `"xla"`: the XLA random number generator (TPU only) + - `"generator"`: an optional `torch.Generator` + synchronized_generator (`torch.Generator`, *optional*): + A random number generator to keep synchronized across processes. + skip_batches (`int`, *optional*, defaults to 0): + The number of batches to skip at the beginning. + use_stateful_dataloader (`bool`, *optional*, defaults to `False`): + Whether to have this class adapt `StatefulDataLoader` from `torchdata` instead of the regular `DataLoader`. + **kwargs (additional keyword arguments, *optional*): + All other keyword arguments to pass to the regular `DataLoader` initialization. + + **Available attributes:** + + - **total_batch_size** (`int`) -- Total batch size of the dataloader across all processes. + Equal to the original batch size when `split_batches=True`; otherwise the original batch size * the total + number of processes + + - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes. + """ + + def __init__( + self, + dataset, + device=None, + rng_types=None, + synchronized_generator=None, + skip_batches=0, + use_stateful_dataloader=False, + _drop_last: bool = False, + _non_blocking: bool = False, + torch_device_mesh=None, + **kwargs, + ): + super().__init__(dataset, use_stateful_dataloader=use_stateful_dataloader, **kwargs) + self.device = device + self.rng_types = rng_types + self.synchronized_generator = synchronized_generator + self.skip_batches = skip_batches + self.gradient_state = GradientState() + self._drop_last = _drop_last + self._non_blocking = _non_blocking + self.iteration = 0 + + def adjust_state_dict_for_prefetch(self): + # DataLoaderShard does not need the DDP prefetch adjustment that DataLoaderDispatcher needs. + # In DataLoaderShard, each process has its own sharded base dataloader and the 1-batch + # look-ahead is already accounted for by the timing of _update_state_dict() calls + # (called before the inner next(), so the captured state already equals the number of + # batches yielded to the user). + pass + + def __iter__(self): + if self.rng_types is not None: + synchronize_rng_states(self.rng_types, self.synchronized_generator) + self.begin() + + self.set_epoch(self.iteration) + dataloader_iter = self.base_dataloader.__iter__() + # We iterate one batch ahead to check when we are at the end + try: + current_batch = next(dataloader_iter) + except StopIteration: + self.end() + return + + batch_index = 0 + while True: + try: + # But we still move it to the device so it is done before `StopIteration` is reached + if self.device is not None: + current_batch = send_to_device(current_batch, self.device, non_blocking=self._non_blocking) + self._update_state_dict() + next_batch = next(dataloader_iter) + if batch_index >= self.skip_batches: + yield current_batch + batch_index += 1 + current_batch = next_batch + except StopIteration: + self.end_of_dataloader = True + self._update_state_dict() + if batch_index >= self.skip_batches: + yield current_batch + break + + self.iteration += 1 + self.end() + + def __reduce__(self): + """ + Define the `__reduce__` method to ensure a `DataLoaderShard` can be pickled and unpickled. This needs to be + explicitly defined since default pickling behavior is broken by `DataLoaderAdapter` messing with its + `__class__` member. + """ + args = super().__reduce__() + return (DataLoaderShard, *args[1:]) + + def set_epoch(self, epoch: int): + # In case it is manually passed in, the user can set it to what they like + if self.iteration != epoch: + self.iteration = epoch + if hasattr(self.batch_sampler, "set_epoch"): + self.batch_sampler.set_epoch(epoch) + if hasattr(self.batch_sampler, "sampler") and hasattr(self.batch_sampler.sampler, "set_epoch"): + self.batch_sampler.sampler.set_epoch(epoch) + if ( + hasattr(self.batch_sampler, "batch_sampler") + and hasattr(self.batch_sampler.batch_sampler, "sampler") + and hasattr(self.batch_sampler.batch_sampler.sampler, "set_epoch") + ): + self.batch_sampler.batch_sampler.sampler.set_epoch(epoch) + # We support if a custom `Dataset` implementation has `set_epoch` + # or in general HF datasets `Datasets` + elif hasattr(self.dataset, "set_epoch"): + self.dataset.set_epoch(epoch) + + @property + def total_batch_size(self): + batch_sampler = self.sampler if isinstance(self.sampler, BatchSampler) else self.batch_sampler + return ( + batch_sampler.batch_size + if getattr(batch_sampler, "split_batches", False) + else (batch_sampler.batch_size * getattr(batch_sampler, "num_processes", 1)) + ) + + @property + def total_dataset_length(self): + if hasattr(self.dataset, "total_length"): + return self.dataset.total_length + else: + return len(self.dataset) + + def get_sampler(self): + return get_sampler(self) + + def set_sampler(self, sampler): + sampler_is_batch_sampler = isinstance(self.sampler, BatchSampler) + if sampler_is_batch_sampler: + self.sampler.sampler = sampler + else: + self.batch_sampler.sampler = sampler + if hasattr(self.batch_sampler, "batch_sampler"): + self.batch_sampler.batch_sampler.sampler = sampler + + +if is_torch_xla_available(): + import torch_xla.distributed.parallel_loader as xpl + + class MpDeviceLoaderWrapper(xpl.MpDeviceLoader): + """ + Wrapper for the xpl.MpDeviceLoader class that knows the total batch size. + + XLA preloading threads will all call DataLoaderShard's __iter__(). Remove rng_types from DataLoaderShard to + prevent it from using the XLA device in the preloading threads, and synchronize the RNG once from the main + thread only. + + **Available attributes:** + + - **total_batch_size** (`int`) -- Total batch size of the dataloader across all processes. + Equal to the original batch size when `split_batches=True`; otherwise the original batch size * the total + number of processes + + - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes. + """ + + def __init__(self, dataloader: DataLoaderShard, device: torch.device): + super().__init__(dataloader, device) + self._rng_types = self._loader.rng_types + self._loader.rng_types = None + self.device = device + + def __iter__(self): + if self._rng_types is not None: + synchronize_rng_states(self._rng_types, self._loader.synchronized_generator) + + return super().__iter__() + + def set_epoch(self, epoch: int): + if hasattr(self.dataloader, "set_epoch"): + self.dataloader.set_epoch(epoch) + + @property + def total_batch_size(self): + return self._loader.total_batch_size + + @property + def total_dataset_length(self): + return self._loader.total_dataset_length + + @property + def batch_sampler(self): + return self._loader.batch_sampler + + @property + def dataloader(self): + return self._loader + + +class DataLoaderDispatcher(DataLoaderAdapter, DataLoaderStateMixin): + """ + Subclass of `DataLoaderAdapter` that will iterate and preprocess on process 0 only, then dispatch on each process + their part of the batch. + + Args: + split_batches (`bool`, *optional*, defaults to `False`): + Whether the resulting `DataLoader` should split the batches of the original data loader across devices or + yield full batches (in which case it will yield batches starting at the `process_index`-th and advancing of + `num_processes` batches at each iteration). Another way to see this is that the observed batch size will be + the same as the initial `dataloader` if this option is set to `True`, the batch size of the initial + `dataloader` multiplied by `num_processes` otherwise. Setting this option to `True` requires that the batch + size of the `dataloader` is a round multiple of `batch_size`. + skip_batches (`int`, *optional*, defaults to 0): + The number of batches to skip at the beginning of an iteration. + use_stateful_dataloader (`bool`, *optional*, defaults to `False`): + Whether to have this class adapt `StatefulDataLoader` from `torchdata` instead of the regular `DataLoader`. + + **Available attributes:** + + - **total_batch_size** (`int`) -- Total batch size of the dataloader across all processes. + Equal to the original batch size when `split_batches=True`; otherwise the original batch size * the total + number of processes + + - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes. + """ + + def __init__( + self, + dataset, + split_batches: bool = False, + skip_batches=0, + use_stateful_dataloader=False, + _drop_last: bool = False, + _non_blocking: bool = False, + slice_fn=None, + torch_device_mesh=None, + **kwargs, + ): + shuffle = False + from torch.utils.data.datapipes.iter.combinatorics import ShufflerIterDataPipe + + # We need to save the shuffling state of the DataPipe + if isinstance(dataset, ShufflerIterDataPipe): + shuffle = dataset._shuffle_enabled + super().__init__(dataset, use_stateful_dataloader=use_stateful_dataloader, **kwargs) + self.split_batches = split_batches + if shuffle: + torch.utils.data.graph_settings.apply_shuffle_settings(dataset, shuffle=shuffle) + + self.gradient_state = GradientState() + self.state = PartialState() + self._drop_last = _drop_last + self._non_blocking = _non_blocking + self.skip_batches = skip_batches + self.torch_device_mesh = torch_device_mesh + + self.slice_fn = slice_tensors if slice_fn is None else slice_fn + self.iteration = 0 + + # if a device mesh is provided extract each dimension (dp, fsdp, tp) + # device mesh may hold any number of dimensions, however, + # below code is for targeted support for dp, fsdp and tp + + # device mesh will be used only if there is tp involved + # or any multi-dimensional parallelism involving tp + # (dp, tp) (fsdp, tp) (dp, fsdp, tp) + # otherwise the default behaviour not using device mesh should be sufficient + # since multi dimensional parallelism devoid of tp would anyway need + # different batches for each process irrespective of dp or fsdp + self.submesh_tp = None + self.submesh_dp = None + self.submesh_fsdp = None + if self.torch_device_mesh and "tp" in self.torch_device_mesh.mesh_dim_names: + self.submesh_tp = self.torch_device_mesh["tp"] + if "dp" in self.torch_device_mesh.mesh_dim_names: + self.submesh_dp = self.torch_device_mesh["dp"] + if "fsdp" in self.torch_device_mesh.mesh_dim_names: + self.submesh_fsdp = self.torch_device_mesh["fsdp"] + if self.submesh_tp and (self.submesh_dp or self.submesh_fsdp): + raise ValueError("TP + (DP/FSDP) is not yet supported in dispatch mode") + + def _fetch_batches(self, iterator): + batches, batch = None, None + # On process 0, we gather the batch to dispatch. + if self.state.process_index == 0: + # Procedure to support TP only is simpler + # since we want to dispatch the same batch of samples across all ranks + # this removes complexity of handling multiple tp rank groups when TP + DP + # combination is involved. + + try: + # for TP case avoid using split_batches + # since it would mean that the dataloader should be spilling out + # duplicates of batches. + if self.split_batches: + # One batch of the main iterator is dispatched and split. + if self.submesh_tp: + logger.warning( + "Use of split_batches for TP would need the dataloader to produce duplicate batches," + "otherwise, use dispatch_batches=True instead." + ) + self._update_state_dict() + batch = next(iterator) + else: + # num_processes batches of the main iterator are concatenated then dispatched and split. + # We add the batches one by one so we have the remainder available when drop_last=False. + batches = [] + if self.submesh_tp: + # when tp, extract single batch and then replicate + self._update_state_dict() + batch = next(iterator) + batches = [batch] * self.state.num_processes + else: + for _ in range(self.state.num_processes): + self._update_state_dict() + batches.append(next(iterator)) + try: + batch = concatenate(batches, dim=0) + except RuntimeError as e: + raise RuntimeError( + "You can't use batches of different size with `dispatch_batches=True` or when using an `IterableDataset`." + "either pass `dispatch_batches=False` and have each process fetch its own batch " + " or pass `split_batches=True`. By doing so, the main process will fetch a full batch and " + "slice it into `num_processes` batches for each process." + ) from e + # In both cases, we need to get the structure of the batch that we will broadcast on other + # processes to initialize the tensors with the right shape. + # data_structure, stop_iteration + batch_info = [get_data_structure(batch), False] + except StopIteration: + batch_info = [None, True] + else: + batch_info = [None, self._stop_iteration] + # This is inplace, so after this instruction, every process has the same `batch_info` as process 0. + broadcast_object_list(batch_info) + self._stop_iteration = batch_info[1] + if self._stop_iteration: + # If drop_last is False and split_batches is False, we may have a remainder to take care of. + if not self.split_batches and not self._drop_last: + if self.state.process_index == 0 and len(batches) > 0: + batch = concatenate(batches, dim=0) + batch_info = [get_data_structure(batch), False] + else: + batch_info = [None, True] + broadcast_object_list(batch_info) + return batch, batch_info + + def __iter__(self): + self.begin() + self.set_epoch(self.iteration) + main_iterator = None + if is_torch_version(">=", "2.0.1"): + # NOTE PyTorch DataLoader adds forward compatibilities for DataPipes, which broadcasts + # shared seed to all dist processes. Thus, we need to create iterator for all dist processes. + # But, we only iterate through the DataLoader on process 0. + main_iterator = self.base_dataloader.__iter__() + elif self.state.process_index == 0: + main_iterator = self.base_dataloader.__iter__() + stop_iteration = False + self._stop_iteration = False + first_batch = None + next_batch, next_batch_info = self._fetch_batches(main_iterator) + batch_index = 0 + while not stop_iteration: + batch, batch_info = next_batch, next_batch_info + + if self.state.process_index != 0: + # Initialize tensors on other processes than process 0. + batch = initialize_tensors(batch_info[0]) + batch = send_to_device(batch, self.state.device, non_blocking=self._non_blocking) + # Broadcast the batch before splitting it. + batch = broadcast(batch, from_process=0) + + if not self._drop_last and first_batch is None: + # We keep at least num processes elements of the first batch to be able to complete the last batch + first_batch = self.slice_fn( + batch, + slice(0, self.state.num_processes), + process_index=self.state.process_index, + num_processes=self.state.num_processes, + ) + + if batch is None: + raise ValueError( + f"Batch does not contain any data (`{batch}`). At the end of all iterable data available before expected stop iteration." + ) + + observed_batch_size = find_batch_size(batch) + batch_size = observed_batch_size // self.state.num_processes + + stop_iteration = self._stop_iteration + if not stop_iteration: + # We may still be at the end of the dataloader without knowing it yet: if there is nothing left in + # the dataloader since the number of batches is a round multiple of the number of processes. + next_batch, next_batch_info = self._fetch_batches(main_iterator) + # next_batch_info[0] is None when there are no more batches, otherwise we still need to process them. + if self._stop_iteration and next_batch_info[0] is None: + stop_iteration = True + + if not self._drop_last and stop_iteration and observed_batch_size % self.state.num_processes != 0: + # If the last batch is not complete, let's add the first batch to it. + batch = concatenate([batch, first_batch], dim=0) + # Batch size computation above is wrong, it's off by 1 so we fix it. + batch_size += 1 + + data_slice = slice(self.state.process_index * batch_size, (self.state.process_index + 1) * batch_size) + batch = self.slice_fn( + batch, + data_slice, + process_index=self.state.process_index, + num_processes=self.state.num_processes, + ) + + if stop_iteration: + self.end_of_dataloader = True + self._update_state_dict() + self.remainder = observed_batch_size + if batch_index >= self.skip_batches: + yield batch + batch_index += 1 + self.iteration += 1 + self.end() + + def set_epoch(self, epoch: int): + # In case it is manually passed in, the user can set it to what they like + if self.iteration != epoch: + self.iteration = epoch + if hasattr(self.batch_sampler, "sampler") and hasattr(self.batch_sampler.sampler, "set_epoch"): + self.batch_sampler.sampler.set_epoch(epoch) + elif hasattr(self.dataset, "set_epoch"): + self.dataset.set_epoch(epoch) + + def __len__(self): + whole_length = len(self.base_dataloader) + if self.split_batches: + return whole_length + elif self._drop_last: + return whole_length // self.state.num_processes + else: + return math.ceil(whole_length / self.state.num_processes) + + def __reduce__(self): + """ + Define the `__reduce__` method to ensure a `DataLoaderDispatcher` can be pickled and unpickled. This needs to + be explicitly defined since default pickling behavior is broken by `DataLoaderAdapter` messing with its + `__class__` member. + """ + args = super().__reduce__() + return (DataLoaderDispatcher, *args[1:]) + + @property + def total_batch_size(self): + return ( + self.dataset.batch_size if self.split_batches else (self.dataset.batch_size * self.dataset.num_processes) + ) + + @property + def total_dataset_length(self): + return len(self.dataset) + + def get_sampler(self): + return get_sampler(self) + + def set_sampler(self, sampler): + sampler_is_batch_sampler = isinstance(self.sampler, BatchSampler) + if sampler_is_batch_sampler: + self.sampler.sampler = sampler + else: + self.batch_sampler.sampler = sampler + if hasattr(self.batch_sampler, "batch_sampler"): + self.batch_sampler.batch_sampler.sampler = sampler + + +def get_sampler(dataloader): + """ + Get the sampler associated to the dataloader + + Args: + dataloader (`torch.utils.data.dataloader.DataLoader`): + The data loader to split across several devices. + Returns: + `torch.utils.data.Sampler`: The sampler associated to the dataloader + """ + sampler_is_batch_sampler = isinstance(dataloader.sampler, BatchSampler) + if sampler_is_batch_sampler: + sampler = getattr(dataloader.sampler, "sampler", None) + else: + sampler = getattr(dataloader.batch_sampler, "sampler", None) + return sampler + + +def prepare_data_loader( + dataloader: DataLoader, + device: Optional[torch.device] = None, + num_processes: Optional[int] = None, + process_index: Optional[int] = None, + split_batches: bool = False, + put_on_device: bool = False, + rng_types: Optional[list[Union[str, RNGType]]] = None, + dispatch_batches: Optional[bool] = None, + even_batches: bool = True, + slice_fn_for_dispatch: Optional[Callable] = None, + use_seedable_sampler: bool = False, + data_seed: Optional[int] = None, + non_blocking: bool = False, + use_stateful_dataloader: bool = False, + torch_device_mesh=None, +) -> DataLoader: + """ + Wraps a PyTorch `DataLoader` to generate batches for one of the processes only. + + Depending on the value of the `drop_last` attribute of the `dataloader` passed, it will either stop the iteration + at the first batch that would be too small / not present on all processes or loop with indices from the beginning. + + Args: + dataloader (`torch.utils.data.dataloader.DataLoader`): + The data loader to split across several devices. + device (`torch.device`): + The target device for the returned `DataLoader`. + num_processes (`int`, *optional*): + The number of processes running concurrently. Will default to the value given by [`~state.PartialState`]. + process_index (`int`, *optional*): + The index of the current process. Will default to the value given by [`~state.PartialState`]. + split_batches (`bool`, *optional*, defaults to `False`): + Whether the resulting `DataLoader` should split the batches of the original data loader across devices or + yield full batches (in which case it will yield batches starting at the `process_index`-th and advancing of + `num_processes` batches at each iteration). + + Another way to see this is that the observed batch size will be the same as the initial `dataloader` if + this option is set to `True`, the batch size of the initial `dataloader` multiplied by `num_processes` + otherwise. + + Setting this option to `True` requires that the batch size of the `dataloader` is a round multiple of + `batch_size`. + put_on_device (`bool`, *optional*, defaults to `False`): + Whether or not to put the batches on `device` (only works if the batches are nested list, tuples or + dictionaries of tensors). + rng_types (list of `str` or [`~utils.RNGType`]): + The list of random number generators to synchronize at the beginning of each iteration. Should be one or + several of: + + - `"torch"`: the base torch random number generator + - `"cuda"`: the CUDA random number generator (GPU only) + - `"xla"`: the XLA random number generator (TPU only) + - `"generator"`: the `torch.Generator` of the sampler (or batch sampler if there is no sampler in your + dataloader) or of the iterable dataset (if it exists) if the underlying dataset is of that type. + + dispatch_batches (`bool`, *optional*): + If set to `True`, the dataloader prepared is only iterated through on the main process and then the batches + are split and broadcast to each process. Will default to `True` when the underlying dataset is an + `IterableDataset`, `False` otherwise. + even_batches (`bool`, *optional*, defaults to `True`): + If set to `True`, in cases where the total batch size across all processes does not exactly divide the + dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among + all workers. + slice_fn_for_dispatch (`Callable`, *optional*`): + If passed, this function will be used to slice tensors across `num_processes`. Will default to + [`~utils.slice_tensors`]. This argument is used only when `dispatch_batches` is set to `True` and will be + ignored otherwise. + use_seedable_sampler (`bool`, *optional*, defaults to `False`): + Whether to use the [`~data_loader.SeedableRandomSampler`] instead of a `RandomSampler` for better + reproducibility. Comes at a cost of potentially different performances due to different shuffling + algorithms but ensures results will be the *exact* same. Should be paired with `set_seed()` at every + `self.set_epoch` + data_seed (`int`, *optional*, defaults to `None`): + The seed to use for the underlying generator when using `use_seedable_sampler`. If `None`, the generator + will use the current default seed from torch. + non_blocking (`bool`, *optional*, defaults to `False`): + If set to `True`, dataloader will utilize non-blocking host-to-device transfers. If the dataloader has + `pin_memory` set to `True`, this will help to increase overlap between data transfer and computations. + use_stateful_dataloader (`bool`, *optional*, defaults to `False`): + "If set to true, the dataloader prepared by the Accelerator will be backed by " + "[torchdata.StatefulDataLoader](https://github.com/pytorch/data/tree/main/torchdata/stateful_dataloader). + This requires `torchdata` version 0.8.0 or higher that supports StatefulDataLoader to be installed." + torch_device_mesh (`torch.distributed.DeviceMesh`, *optional*, defaults to `None`): + PyTorch device mesh. + + + Returns: + `torch.utils.data.dataloader.DataLoader`: A new data loader that will yield the portion of the batches + + + + `BatchSampler`s with varying batch sizes are not enabled by default. To enable this behaviour, set `even_batches` + equal to `False` + + + """ + if dispatch_batches is None: + if not put_on_device: + dispatch_batches = False + else: + dispatch_batches = isinstance(dataloader.dataset, IterableDataset) + + if dispatch_batches and not put_on_device: + raise ValueError("Using `dispatch_batches=True` requires `put_on_device=True`.") + # Grab defaults from PartialState + state = PartialState() + if num_processes is None: + num_processes = state.num_processes + + if process_index is None: + process_index = state.process_index + + if torch_device_mesh: + if state.distributed_type == DistributedType.DEEPSPEED: + # In DeepSpeed, the optimizer sharing level in DP is determined by the config file. + # Only considers "dp" and "tp". + # Given a device mesh (dp, tp) = (2, 3): + # - From the data parallel perspective, ranks should be structured as: 0 0 0 1 1 1 + # - Processes with the same DP rank will receive the same batch. + submesh_tp_size = 1 + if "tp" in torch_device_mesh.mesh_dim_names: + submesh_tp_size = torch_device_mesh["tp"].size() + process_index = process_index // submesh_tp_size + num_processes = num_processes // submesh_tp_size + else: + # when device mesh is used, specifically with TP + # then there is need to update process_index and num_processes + # to bring in the effect of generating same batch across TP ranks + # and different batch across FSDP and DP ranks. + # Example: + # if device mesh is (dp,fsdp,tp) = (2, 2, 3) + # ranks would range from 0...11 + # from data angle ranks should look like 0 0 0 1 1 1 2 2 2 3 3 3 + # processes with same ranks/ids would receive the same batch + # for CP the same as TP applies + submesh_fsdp_size = 1 + submesh_dp_size = 1 + submesh_tp_size = 1 + submesh_cp_size = 1 + if "tp" in torch_device_mesh.mesh_dim_names: + submesh_tp_size = torch_device_mesh["tp"].size() + if "cp" in torch_device_mesh.mesh_dim_names: + submesh_cp_size = torch_device_mesh["cp"].size() + if "dp_replicate" in torch_device_mesh.mesh_dim_names: + submesh_dp_size = torch_device_mesh["dp_replicate"].size() + if "dp_shard" in torch_device_mesh.mesh_dim_names: + submesh_fsdp_size = torch_device_mesh["dp_shard"].size() + process_index = process_index // (submesh_tp_size * submesh_cp_size) + num_processes = submesh_fsdp_size * submesh_dp_size + + # Sanity check + if split_batches: + if dataloader.batch_size is not None: + batch_size_for_check = dataloader.batch_size + else: + # For custom batch_sampler + if hasattr(dataloader.batch_sampler, "batch_size"): + batch_size_for_check = dataloader.batch_sampler.batch_size + else: + raise ValueError( + "In order to use `split_batches==True` you must have a `batch_size` attribute either in the passed " + "`dataloader` or `dataloader.batch_sampler` objects, and it has to return a natural number. " + "Your `dataloader.batch_size` is None and `dataloader.batch_sampler` " + f"(`{type(dataloader.batch_sampler)}`) does not have the `batch_size` attribute set." + ) + + if batch_size_for_check > 1 and batch_size_for_check % num_processes != 0: + raise ValueError( + f"To use a `DataLoader` in `split_batches` mode, the batch size ({dataloader.batch_size}) " + f"needs to be a round multiple of the number of processes ({num_processes})." + ) + + new_dataset = dataloader.dataset + # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it + new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None + sampler_is_batch_sampler = isinstance(dataloader.sampler, BatchSampler) + synchronized_generator = None + + sampler = get_sampler(dataloader) + if isinstance(sampler, RandomSampler) and use_seedable_sampler: + # When iterating through the dataloader during distributed processes + # we want to ensure that on each process we are iterating through the same + # samples in the same order if a seed is set. This requires a tweak + # to the `torch.utils.data.RandomSampler` class (if used). + sampler = SeedableRandomSampler( + data_source=sampler.data_source, + replacement=sampler.replacement, + num_samples=sampler._num_samples, + generator=getattr( + sampler, + "generator", + torch.Generator(device=torch.get_default_device() if hasattr(torch, "get_default_device") else "cpu"), + ), + data_seed=data_seed, + ) + + if isinstance(dataloader.sampler, RandomSampler) and state.distributed_type == DistributedType.XLA: + # isinstance(dataloader.sampler, RandomSampler) indicates the original dataloader has `shuffle` enabled. + generator = torch.Generator( + device=torch.get_default_device() if hasattr(torch, "get_default_device") else "cpu" + ) + seed = int(torch.empty((), dtype=torch.int64).random_().item()) + generator.manual_seed(seed) + dataloader.generator = generator + dataloader.sampler.generator = generator + # No change if no multiprocess + if (num_processes != 1 or state.distributed_type == DistributedType.MEGATRON_LM) and not dispatch_batches: + if is_datasets_available(): + from datasets import IterableDataset as DatasetsIterableDataset + if ( + is_datasets_available() + and isinstance(new_dataset, DatasetsIterableDataset) + and not split_batches + and new_dataset.n_shards >= num_processes + ): + new_dataset = new_dataset.shard(num_shards=num_processes, index=process_index) + elif isinstance(new_dataset, IterableDataset): + if getattr(dataloader.dataset, "generator", None) is not None: + synchronized_generator = dataloader.dataset.generator + new_dataset = IterableDatasetShard( + new_dataset, + batch_size=dataloader.batch_size, + drop_last=dataloader.drop_last, + num_processes=num_processes, + process_index=process_index, + split_batches=split_batches, + ) + else: + if not use_seedable_sampler and hasattr(sampler, "generator"): + if sampler.generator is None: + sampler.generator = torch.Generator( + device=torch.get_default_device() if hasattr(torch, "get_default_device") else "cpu" + ) + seed = int(torch.empty((), dtype=torch.int64).random_().item()) + sampler.generator.manual_seed(seed) + synchronized_generator = sampler.generator + batch_sampler = dataloader.sampler if sampler_is_batch_sampler else dataloader.batch_sampler + new_batch_sampler = BatchSamplerShard( + batch_sampler, + num_processes=num_processes, + process_index=process_index, + split_batches=split_batches, + even_batches=even_batches, + ) + + # We ignore all of those since they are all dealt with by our new_batch_sampler + ignore_kwargs = [ + "batch_size", + "shuffle", + "sampler", + "batch_sampler", + "drop_last", + ] + + if rng_types is not None and synchronized_generator is None and "generator" in rng_types: + rng_types.remove("generator") + + kwargs = { + k: getattr(dataloader, k, _PYTORCH_DATALOADER_KWARGS[k]) + for k in _PYTORCH_DATALOADER_KWARGS + if k not in ignore_kwargs + } + + # Need to provide batch_size as batch_sampler is None for Iterable dataset + if new_batch_sampler is None: + kwargs["drop_last"] = dataloader.drop_last + kwargs["batch_size"] = ( + dataloader.batch_size // num_processes if split_batches and not dispatch_batches else dataloader.batch_size + ) + if dispatch_batches: + kwargs.pop("generator") + dataloader = DataLoaderDispatcher( + new_dataset, + split_batches=split_batches, + batch_sampler=new_batch_sampler, + _drop_last=dataloader.drop_last, + _non_blocking=non_blocking, + slice_fn=slice_fn_for_dispatch, + use_stateful_dataloader=use_stateful_dataloader, + torch_device_mesh=torch_device_mesh, + **kwargs, + ) + elif sampler_is_batch_sampler: + dataloader = DataLoaderShard( + new_dataset, + device=device if put_on_device and state.distributed_type != DistributedType.XLA else None, + sampler=new_batch_sampler, + batch_size=dataloader.batch_size, + rng_types=rng_types, + _drop_last=dataloader.drop_last, + _non_blocking=non_blocking, + synchronized_generator=synchronized_generator, + use_stateful_dataloader=use_stateful_dataloader, + **kwargs, + ) + else: + dataloader = DataLoaderShard( + new_dataset, + device=device if put_on_device and state.distributed_type != DistributedType.XLA else None, + batch_sampler=new_batch_sampler, + rng_types=rng_types, + synchronized_generator=synchronized_generator, + _drop_last=dataloader.drop_last, + _non_blocking=non_blocking, + use_stateful_dataloader=use_stateful_dataloader, + **kwargs, + ) + + if isinstance(sampler, SeedableRandomSampler) and use_seedable_sampler: + dataloader.set_sampler(sampler) + if state.distributed_type == DistributedType.XLA: + return MpDeviceLoaderWrapper(dataloader, device) + return dataloader + + +class SkipBatchSampler(BatchSampler): + """ + A `torch.utils.data.BatchSampler` that skips the first `n` batches of another `torch.utils.data.BatchSampler`. + Should not be used if the original dataloader is a `StatefulDataLoader`. + """ + + def __init__(self, batch_sampler, skip_batches=0): + self.batch_sampler = batch_sampler + self.skip_batches = skip_batches + + def __iter__(self): + for index, samples in enumerate(self.batch_sampler): + if index >= self.skip_batches: + yield samples + + @property + def total_length(self): + return len(self.batch_sampler) + + def __len__(self): + return len(self.batch_sampler) - self.skip_batches + + +class SkipDataLoader(DataLoaderAdapter, DataLoaderStateMixin): + """ + Subclass of a PyTorch `DataLoader` that will skip the first batches. Generally it's preferable to use + `skip_first_batches`/`torchdata.StatefulDataLoader` instead of this class. + + Args: + dataset (`torch.utils.data.dataset.Dataset`): + The dataset to use to build this dataloader. + skip_batches (`int`, *optional*, defaults to 0): + The number of batches to skip at the beginning. + kwargs: + All other keyword arguments to pass to the regular `DataLoader` initialization. + """ + + def __init__(self, dataset, skip_batches=0, use_stateful_dataloader=False, **kwargs): + super().__init__(dataset, use_stateful_dataloader=use_stateful_dataloader, **kwargs) + self.skip_batches = skip_batches + self.gradient_state = GradientState() + + def __iter__(self): + self.begin() + for index, batch in enumerate(self.base_dataloader.__iter__()): + if index >= self.skip_batches: + self._update_state_dict() + yield batch + self.end() + + def __len__(self): + return len(self.base_dataloader) - self.skip_batches + + def __reduce__(self): + """ + Define the `__reduce__` method to ensure a `SkipDataLoader` can be pickled and unpickled. This needs to be + explicitly defined since default pickling behavior is broken by `DataLoaderAdapter` messing with its + `__class__` member. + """ + args = super().__reduce__() + return (SkipDataLoader, *args[1:]) + + +def skip_first_batches(dataloader, num_batches=0): + """ + Creates a `torch.utils.data.DataLoader` that will efficiently skip the first `num_batches`. Should not be used if + the original dataloader is a `StatefulDataLoader`. + """ + state = PartialState() + if state.distributed_type == DistributedType.XLA: + device = dataloader.device + dataloader = dataloader.dataloader + + dataset = dataloader.dataset + sampler_is_batch_sampler = False + if isinstance(dataset, IterableDataset): + new_batch_sampler = None + else: + sampler_is_batch_sampler = isinstance(dataloader.sampler, BatchSampler) + batch_sampler = dataloader.sampler if sampler_is_batch_sampler else dataloader.batch_sampler + new_batch_sampler = SkipBatchSampler(batch_sampler, skip_batches=num_batches) + + # We ignore all of those since they are all dealt with by our new_batch_sampler + ignore_kwargs = [ + "batch_size", + "shuffle", + "sampler", + "batch_sampler", + "drop_last", + ] + + kwargs = { + k: getattr(dataloader, k, _PYTORCH_DATALOADER_KWARGS[k]) + for k in _PYTORCH_DATALOADER_KWARGS + if k not in ignore_kwargs + } + + # Need to provide batch_size as batch_sampler is None for Iterable dataset + if new_batch_sampler is None: + kwargs["drop_last"] = dataloader.drop_last + kwargs["batch_size"] = dataloader.batch_size + + if isinstance(dataloader, DataLoaderDispatcher): + if new_batch_sampler is None: + # Need to manually skip batches in the dataloader + kwargs["skip_batches"] = num_batches + dataloader = DataLoaderDispatcher( + dataset, + split_batches=dataloader.split_batches, + batch_sampler=new_batch_sampler, + _drop_last=dataloader._drop_last, + **kwargs, + ) + elif isinstance(dataloader, DataLoaderShard): + if new_batch_sampler is None: + # Need to manually skip batches in the dataloader + kwargs["skip_batches"] = num_batches + elif sampler_is_batch_sampler: + kwargs["sampler"] = new_batch_sampler + kwargs["batch_size"] = dataloader.batch_size + else: + kwargs["batch_sampler"] = new_batch_sampler + dataloader = DataLoaderShard( + dataset, + device=dataloader.device, + rng_types=dataloader.rng_types, + synchronized_generator=dataloader.synchronized_generator, + **kwargs, + ) + else: + if new_batch_sampler is None: + # Need to manually skip batches in the dataloader + dataloader = SkipDataLoader(dataset, skip_batches=num_batches, **kwargs) + else: + dataloader = DataLoader(dataset, batch_sampler=new_batch_sampler, **kwargs) + + if state.distributed_type == DistributedType.XLA: + dataloader = MpDeviceLoaderWrapper(dataloader, device) + + return dataloader diff --git a/kbidu_env/Lib/site-packages/accelerate/hooks.py b/kbidu_env/Lib/site-packages/accelerate/hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..272bfb736b89c50c6f3fc1189ecf29bbb122172b --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/hooks.py @@ -0,0 +1,810 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +from collections.abc import Mapping +from typing import Optional, Union + +import torch +import torch.nn as nn + +from .state import PartialState +from .utils import ( + PrefixedDataset, + find_device, + named_module_tensors, + send_to_device, + set_module_tensor_to_device, +) +from .utils.imports import ( + is_mlu_available, + is_musa_available, + is_npu_available, +) +from .utils.memory import clear_device_cache +from .utils.modeling import get_non_persistent_buffers +from .utils.other import recursive_getattr + + +def _compiler_disable(fn): + """ + Lazy version of `torch.compiler.disable` that avoids importing `torch._dynamo` at decoration time. + `torch.compiler.disable` eagerly imports `torch._dynamo` which adds ~4s to import time. + """ + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + if not hasattr(wrapper, "_compiled_fn"): + wrapper._compiled_fn = torch.compiler.disable(fn) + return wrapper._compiled_fn(*args, **kwargs) + + return wrapper + + +_accelerate_added_attributes = ["to", "cuda", "npu", "xpu", "mlu", "sdaa", "musa"] + + +class ModelHook: + """ + A hook that contains callbacks to be executed just before and after the forward method of a model. The difference + with PyTorch existing hooks is that they get passed along the kwargs. + + Class attribute: + - **no_grad** (`bool`, *optional*, defaults to `False`) -- Whether or not to execute the actual forward pass under + the `torch.no_grad()` context manager. + """ + + no_grad = False + + def init_hook(self, module): + """ + To be executed when the hook is attached to the module. + + Args: + module (`torch.nn.Module`): The module attached to this hook. + """ + return module + + def pre_forward(self, module, *args, **kwargs): + """ + To be executed just before the forward method of the model. + + Args: + module (`torch.nn.Module`): The module whose forward pass will be executed just after this event. + args (`Tuple[Any]`): The positional arguments passed to the module. + kwargs (`Dict[Str, Any]`): The keyword arguments passed to the module. + + Returns: + `Tuple[Tuple[Any], Dict[Str, Any]]`: A tuple with the treated `args` and `kwargs`. + """ + return args, kwargs + + def post_forward(self, module, output): + """ + To be executed just after the forward method of the model. + + Args: + module (`torch.nn.Module`): The module whose forward pass been executed just before this event. + output (`Any`): The output of the module. + + Returns: + `Any`: The processed `output`. + """ + return output + + def detach_hook(self, module): + """ + To be executed when the hook is detached from a module. + + Args: + module (`torch.nn.Module`): The module detached from this hook. + """ + return module + + +class SequentialHook(ModelHook): + """ + A hook that can contain several hooks and iterates through them at each event. + """ + + def __init__(self, *hooks): + self.hooks = hooks + + def init_hook(self, module): + for hook in self.hooks: + module = hook.init_hook(module) + return module + + @_compiler_disable + def pre_forward(self, module, *args, **kwargs): + for hook in self.hooks: + args, kwargs = hook.pre_forward(module, *args, **kwargs) + return args, kwargs + + @_compiler_disable + def post_forward(self, module, output): + for hook in self.hooks: + output = hook.post_forward(module, output) + return output + + def detach_hook(self, module): + for hook in self.hooks: + module = hook.detach_hook(module) + return module + + +def add_hook_to_module(module: nn.Module, hook: ModelHook, append: bool = False): + """ + Adds a hook to a given module. This will rewrite the `forward` method of the module to include the hook, to remove + this behavior and restore the original `forward` method, use `remove_hook_from_module`. + + + + If the module already contains a hook, this will replace it with the new hook passed by default. To chain two hooks + together, pass `append=True`, so it chains the current and new hook into an instance of the `SequentialHook` class. + + + + Args: + module (`torch.nn.Module`): + The module to attach a hook to. + hook (`ModelHook`): + The hook to attach. + append (`bool`, *optional*, defaults to `False`): + Whether the hook should be chained with an existing one (if module already contains a hook) or not. + + Returns: + `torch.nn.Module`: The same module, with the hook attached (the module is modified in place, so the result can + be discarded). + """ + if append and (getattr(module, "_hf_hook", None) is not None): + old_hook = module._hf_hook + remove_hook_from_module(module) + hook = SequentialHook(old_hook, hook) + + if hasattr(module, "_hf_hook") and hasattr(module, "_old_forward"): + # If we already put some hook on this module, we replace it with the new one. + old_forward = module._old_forward + else: + old_forward = module.forward + module._old_forward = old_forward + + module = hook.init_hook(module) + module._hf_hook = hook + + def new_forward(module, *args, **kwargs): + args, kwargs = module._hf_hook.pre_forward(module, *args, **kwargs) + if module._hf_hook.no_grad: + with torch.no_grad(): + output = module._old_forward(*args, **kwargs) + else: + output = module._old_forward(*args, **kwargs) + return module._hf_hook.post_forward(module, output) + + # Overriding a GraphModuleImpl forward freezes the forward call and later modifications on the graph will fail. + # Reference: https://pytorch.slack.com/archives/C3PDTEV8E/p1705929610405409 + if "GraphModuleImpl" in str(type(module)): + module.__class__.forward = functools.update_wrapper(functools.partial(new_forward, module), old_forward) + else: + module.forward = functools.update_wrapper(functools.partial(new_forward, module), old_forward) + + return module + + +def remove_hook_from_module(module: nn.Module, recurse=False): + """ + Removes any hook attached to a module via `add_hook_to_module`. + + Args: + module (`torch.nn.Module`): The module to attach a hook to. + recurse (`bool`, **optional**): Whether to remove the hooks recursively + + Returns: + `torch.nn.Module`: The same module, with the hook detached (the module is modified in place, so the result can + be discarded). + """ + + if hasattr(module, "_hf_hook"): + module._hf_hook.detach_hook(module) + delattr(module, "_hf_hook") + + if hasattr(module, "_old_forward"): + # Overriding a GraphModuleImpl forward freezes the forward call and later modifications on the graph will fail. + # Reference: https://pytorch.slack.com/archives/C3PDTEV8E/p1705929610405409 + if "GraphModuleImpl" in str(type(module)): + module.__class__.forward = module._old_forward + else: + module.forward = module._old_forward + delattr(module, "_old_forward") + + # Remove accelerate added warning hooks from dispatch_model + for attr in _accelerate_added_attributes: + module.__dict__.pop(attr, None) + + if recurse: + for child in module.children(): + remove_hook_from_module(child, recurse) + + return module + + +class AlignDevicesHook(ModelHook): + """ + A generic `ModelHook` that ensures inputs and model weights are on the same device for the forward pass of the + associated module, potentially offloading the weights after the forward pass. + + Args: + execution_device (`torch.device`, *optional*): + The device on which inputs and model weights should be placed before the forward pass. + offload (`bool`, *optional*, defaults to `False`): + Whether or not the weights should be offloaded after the forward pass. + io_same_device (`bool`, *optional*, defaults to `False`): + Whether or not the output should be placed on the same device as the input was. + weights_map (`Mapping[str, torch.Tensor]`, *optional*): + When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values. + offload_buffers (`bool`, *optional*, defaults to `False`): + Whether or not to include the associated module's buffers when offloading. + place_submodules (`bool`, *optional*, defaults to `False`): + Whether to place the submodules on `execution_device` during the `init_hook` event. + """ + + def __init__( + self, + execution_device: Optional[Union[int, str, torch.device]] = None, + offload: bool = False, + io_same_device: bool = False, + weights_map: Optional[Mapping] = None, + offload_buffers: bool = False, + place_submodules: bool = False, + skip_keys: Optional[Union[str, list[str]]] = None, + tied_params_map: Optional[dict[int, dict[torch.device, torch.Tensor]]] = None, + ): + self.execution_device = execution_device + self.offload = offload + self.io_same_device = io_same_device + self.weights_map = weights_map + self.offload_buffers = offload_buffers + self.place_submodules = place_submodules + self.skip_keys = skip_keys + + # Will contain the input device when `io_same_device=True`. + self.input_device = None + self.param_original_devices = {} + self.buffer_original_devices = {} + self.tied_params_names = set() + + # The hook pre_forward/post_forward need to have knowledge of this dictionary, as with offloading we want to avoid duplicating memory + # for tied weights already loaded on the target execution device. + self.tied_params_map = tied_params_map + + def __repr__(self): + return ( + f"AlignDevicesHook(execution_device={self.execution_device}, offload={self.offload}, " + f"io_same_device={self.io_same_device}, offload_buffers={self.offload_buffers}, " + f"place_submodules={self.place_submodules}, skip_keys={repr(self.skip_keys)})" + ) + + def init_hook(self, module): + # In case the AlignDevicesHook is on meta device, ignore tied weights as data_ptr() is then always zero. + if self.execution_device == "meta" or self.execution_device == torch.device("meta"): + self.tied_params_map = None + + if not self.offload and self.execution_device is not None: + for name, _ in named_module_tensors(module, recurse=self.place_submodules): + set_module_tensor_to_device(module, name, self.execution_device, tied_params_map=self.tied_params_map) + elif self.offload: + self.original_devices = { + name: param.device for name, param in named_module_tensors(module, recurse=self.place_submodules) + } + if self.weights_map is None: + self.weights_map = { + name: param.to("cpu") + for name, param in named_module_tensors( + module, include_buffers=self.offload_buffers, recurse=self.place_submodules + ) + } + for name, _ in named_module_tensors( + module, include_buffers=self.offload_buffers, recurse=self.place_submodules, remove_non_persistent=True + ): + # When using disk offloading, we can not rely on `weights_map[name].data_ptr()` as the reference pointer, + # as we have no guarantee that safetensors' `file.get_tensor()` will always give the same pointer. + # As we have no reliable way to track the shared data pointer of tied weights in this case, we use tied_params_names: List[str] + # to add on the fly pointers to `tied_params_map` in the pre_forward call. + if ( + self.tied_params_map is not None + and recursive_getattr(module, name).data_ptr() in self.tied_params_map + ): + self.tied_params_names.add(name) + + set_module_tensor_to_device(module, name, "meta") + + if not self.offload_buffers and self.execution_device is not None: + for name, _ in module.named_buffers(recurse=self.place_submodules): + set_module_tensor_to_device( + module, name, self.execution_device, tied_params_map=self.tied_params_map + ) + elif self.offload_buffers and self.execution_device is not None: + for name in get_non_persistent_buffers(module, recurse=self.place_submodules): + set_module_tensor_to_device( + module, name, self.execution_device, tied_params_map=self.tied_params_map + ) + + return module + + def _maybe_get_fp16_statistics(self, name, value): + # Some quantized weights keep scale statistics as separate state-dict entries rather than + # parameters or buffers. When materializing an int8 weight from `weights_map`, pass those + # statistics along so the restored parameter does not keep stale meta-device attributes. + if value is None or value.dtype != torch.int8 or "weight" not in name: + return None + + statistics_name = name.replace("weight", "SCB") + if statistics_name in self.weights_map: + return self.weights_map[statistics_name] + + return None + + @_compiler_disable + def pre_forward(self, module, *args, **kwargs): + if self.io_same_device: + self.input_device = find_device([args, kwargs]) + if self.offload: + self.tied_pointers_to_remove = set() + + for name, _ in named_module_tensors( + module, + include_buffers=self.offload_buffers, + recurse=self.place_submodules, + remove_non_persistent=True, + ): + value = self.weights_map[name] + fp16_statistics = self._maybe_get_fp16_statistics(name, value) + + # In case we are using offloading with tied weights, we need to keep track of the offloaded weights + # that are loaded on device at this point, as we will need to remove them as well from the dictionary + # self.tied_params_map in order to allow to free memory. + if name in self.tied_params_names and value.data_ptr() not in self.tied_params_map: + self.tied_params_map[value.data_ptr()] = {} + + if ( + value is not None + and self.tied_params_map is not None + and value.data_ptr() in self.tied_params_map + and self.execution_device not in self.tied_params_map[value.data_ptr()] + ): + self.tied_pointers_to_remove.add((value.data_ptr(), self.execution_device)) + + set_module_tensor_to_device( + module, + name, + self.execution_device, + value=value, + fp16_statistics=fp16_statistics, + tied_params_map=self.tied_params_map, + ) + + return send_to_device(args, self.execution_device), send_to_device( + kwargs, self.execution_device, skip_keys=self.skip_keys + ) + + @_compiler_disable + def post_forward(self, module, output): + if self.offload: + for name, _ in named_module_tensors( + module, + include_buffers=self.offload_buffers, + recurse=self.place_submodules, + remove_non_persistent=True, + ): + set_module_tensor_to_device(module, name, "meta") + if type(module).__name__ == "Linear8bitLt": + module.state.SCB = None + module.state.CxB = None + + # We may have loaded tied weights into self.tied_params_map (avoiding to load them several times in e.g. submodules): remove them from + # this dictionary to allow the garbage collector to do its job. + for value_pointer, device in self.tied_pointers_to_remove: + if isinstance(device, int): + if is_npu_available(): + device = f"npu:{device}" + elif is_mlu_available(): + device = f"mlu:{device}" + elif is_musa_available(): + device = f"musa:{device}" + if device in self.tied_params_map[value_pointer]: + del self.tied_params_map[value_pointer][device] + self.tied_pointers_to_remove = set() + if self.io_same_device and self.input_device is not None: + output = send_to_device(output, self.input_device, skip_keys=self.skip_keys) + + return output + + def detach_hook(self, module): + if self.offload: + for name, device in self.original_devices.items(): + if device != torch.device("meta"): + value = self.weights_map.get(name, None) + fp16_statistics = self._maybe_get_fp16_statistics(name, value) + set_module_tensor_to_device(module, name, device, value=value, fp16_statistics=fp16_statistics) + return module + + +def attach_execution_device_hook( + module: torch.nn.Module, + execution_device: Union[int, str, torch.device], + skip_keys: Optional[Union[str, list[str]]] = None, + preload_module_classes: Optional[list[str]] = None, + tied_params_map: Optional[dict[int, dict[torch.device, torch.Tensor]]] = None, +): + """ + Recursively attaches `AlignDevicesHook` to all submodules of a given model to make sure they have the right + execution device + + Args: + module (`torch.nn.Module`): + The module where we want to attach the hooks. + execution_device (`int`, `str` or `torch.device`): + The device on which inputs and model weights should be placed before the forward pass. + skip_keys (`str` or `List[str]`, *optional*): + A list of keys to ignore when moving inputs or outputs between devices. + preload_module_classes (`List[str]`, *optional*): + A list of classes whose instances should load all their weights (even in the submodules) at the beginning + of the forward. This should only be used for classes that have submodules which are registered but not + called directly during the forward, for instance if a `dense` linear layer is registered, but at forward, + `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly. + tied_params_map (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], *optional*, defaults to `None`): + A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution + device, this parameter is useful to reuse the first available pointer of a shared weight for all others, + instead of duplicating memory. + """ + if not hasattr(module, "_hf_hook") and len(module.state_dict()) > 0: + add_hook_to_module( + module, + AlignDevicesHook(execution_device, skip_keys=skip_keys, tied_params_map=tied_params_map), + ) + + # Break the recursion if we get to a preload module. + if preload_module_classes is not None and module.__class__.__name__ in preload_module_classes: + return + + for child in module.children(): + attach_execution_device_hook( + child, + execution_device, + skip_keys=skip_keys, + preload_module_classes=preload_module_classes, + tied_params_map=tied_params_map, + ) + + +def attach_align_device_hook( + module: torch.nn.Module, + execution_device: Optional[torch.device] = None, + offload: bool = False, + weights_map: Optional[Mapping] = None, + offload_buffers: bool = False, + module_name: str = "", + skip_keys: Optional[Union[str, list[str]]] = None, + preload_module_classes: Optional[list[str]] = None, + tied_params_map: Optional[dict[int, dict[torch.device, torch.Tensor]]] = None, +): + """ + Recursively attaches `AlignDevicesHook` to all submodules of a given model that have direct parameters and/or + buffers. + + Args: + module (`torch.nn.Module`): + The module where we want to attach the hooks. + execution_device (`torch.device`, *optional*): + The device on which inputs and model weights should be placed before the forward pass. + offload (`bool`, *optional*, defaults to `False`): + Whether or not the weights should be offloaded after the forward pass. + weights_map (`Mapping[str, torch.Tensor]`, *optional*): + When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values. + offload_buffers (`bool`, *optional*, defaults to `False`): + Whether or not to include the associated module's buffers when offloading. + module_name (`str`, *optional*, defaults to `""`): + The name of the module. + skip_keys (`str` or `List[str]`, *optional*): + A list of keys to ignore when moving inputs or outputs between devices. + preload_module_classes (`List[str]`, *optional*): + A list of classes whose instances should load all their weights (even in the submodules) at the beginning + of the forward. This should only be used for classes that have submodules which are registered but not + called directly during the forward, for instance if a `dense` linear layer is registered, but at forward, + `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly. + tied_params_map (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], *optional*, defaults to `None`): + A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution + device, this parameter is useful to reuse the first available pointer of a shared weight for all others, + instead of duplicating memory. + """ + # Attach the hook on this module if it has any direct tensor. + directs = named_module_tensors(module) + full_offload = ( + offload and preload_module_classes is not None and module.__class__.__name__ in preload_module_classes + ) + + if len(list(directs)) > 0 or full_offload: + if weights_map is not None: + prefix = f"{module_name}." if len(module_name) > 0 else "" + prefixed_weights_map = PrefixedDataset(weights_map, prefix) + else: + prefixed_weights_map = None + hook = AlignDevicesHook( + execution_device=execution_device, + offload=offload, + weights_map=prefixed_weights_map, + offload_buffers=offload_buffers, + place_submodules=full_offload, + skip_keys=skip_keys, + tied_params_map=tied_params_map, + ) + add_hook_to_module(module, hook, append=True) + + # We stop the recursion in case we hit the full offload. + if full_offload: + return + + # Recurse on all children of the module. + for child_name, child in module.named_children(): + child_name = f"{module_name}.{child_name}" if len(module_name) > 0 else child_name + attach_align_device_hook( + child, + execution_device=execution_device, + offload=offload, + weights_map=weights_map, + offload_buffers=offload_buffers, + module_name=child_name, + preload_module_classes=preload_module_classes, + skip_keys=skip_keys, + tied_params_map=tied_params_map, + ) + + +def remove_hook_from_submodules(module: nn.Module): + """ + Recursively removes all hooks attached on the submodules of a given model. + + Args: + module (`torch.nn.Module`): The module on which to remove all hooks. + """ + remove_hook_from_module(module) + for child in module.children(): + remove_hook_from_submodules(child) + + +def attach_align_device_hook_on_blocks( + module: nn.Module, + execution_device: Optional[Union[torch.device, dict[str, torch.device]]] = None, + offload: Union[bool, dict[str, bool]] = False, + weights_map: Optional[Mapping] = None, + offload_buffers: bool = False, + module_name: str = "", + skip_keys: Optional[Union[str, list[str]]] = None, + preload_module_classes: Optional[list[str]] = None, + tied_params_map: Optional[dict[int, dict[torch.device, torch.Tensor]]] = None, +): + """ + Attaches `AlignDevicesHook` to all blocks of a given model as needed. + + Args: + module (`torch.nn.Module`): + The module where we want to attach the hooks. + execution_device (`torch.device` or `Dict[str, torch.device]`, *optional*): + The device on which inputs and model weights should be placed before the forward pass. It can be one device + for the whole module, or a dictionary mapping module name to device. + offload (`bool`, *optional*, defaults to `False`): + Whether or not the weights should be offloaded after the forward pass. It can be one boolean for the whole + module, or a dictionary mapping module name to boolean. + weights_map (`Mapping[str, torch.Tensor]`, *optional*): + When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values. + offload_buffers (`bool`, *optional*, defaults to `False`): + Whether or not to include the associated module's buffers when offloading. + module_name (`str`, *optional*, defaults to `""`): + The name of the module. + skip_keys (`str` or `List[str]`, *optional*): + A list of keys to ignore when moving inputs or outputs between devices. + preload_module_classes (`List[str]`, *optional*): + A list of classes whose instances should load all their weights (even in the submodules) at the beginning + of the forward. This should only be used for classes that have submodules which are registered but not + called directly during the forward, for instance if a `dense` linear layer is registered, but at forward, + `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly. + tied_params_map (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], *optional*, defaults to `None`): + A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution + device, this parameter is useful to reuse the first available pointer of a shared weight for all others, + instead of duplicating memory. + """ + # If one device and one offload, we've got one hook. + if not isinstance(execution_device, Mapping) and not isinstance(offload, dict): + if not offload: + hook = AlignDevicesHook( + execution_device=execution_device, + io_same_device=True, + skip_keys=skip_keys, + place_submodules=True, + tied_params_map=tied_params_map, + ) + add_hook_to_module(module, hook) + else: + attach_align_device_hook( + module, + execution_device=execution_device, + offload=True, + weights_map=weights_map, + offload_buffers=offload_buffers, + module_name=module_name, + skip_keys=skip_keys, + tied_params_map=tied_params_map, + ) + return + + if not isinstance(execution_device, Mapping): + execution_device = {key: execution_device for key in offload.keys()} + if not isinstance(offload, Mapping): + offload = {key: offload for key in execution_device.keys()} + + if module_name in execution_device and module_name in offload and not offload[module_name]: + hook = AlignDevicesHook( + execution_device=execution_device[module_name], + offload_buffers=offload_buffers, + io_same_device=(module_name == ""), + place_submodules=True, + skip_keys=skip_keys, + tied_params_map=tied_params_map, + ) + add_hook_to_module(module, hook) + attach_execution_device_hook( + module, execution_device[module_name], skip_keys=skip_keys, tied_params_map=tied_params_map + ) + elif module_name in execution_device and module_name in offload: + attach_align_device_hook( + module, + execution_device=execution_device[module_name], + offload=True, + weights_map=weights_map, + offload_buffers=offload_buffers, + module_name=module_name, + skip_keys=skip_keys, + preload_module_classes=preload_module_classes, + tied_params_map=tied_params_map, + ) + if not hasattr(module, "_hf_hook"): + hook = AlignDevicesHook( + execution_device=execution_device[module_name], + io_same_device=(module_name == ""), + skip_keys=skip_keys, + tied_params_map=tied_params_map, + ) + add_hook_to_module(module, hook) + attach_execution_device_hook( + module, + execution_device[module_name], + preload_module_classes=preload_module_classes, + skip_keys=skip_keys, + tied_params_map=tied_params_map, + ) + elif module_name == "": + hook = AlignDevicesHook( + execution_device=execution_device.get(""), + io_same_device=True, + skip_keys=skip_keys, + tied_params_map=tied_params_map, + ) + add_hook_to_module(module, hook) + + for child_name, child in module.named_children(): + child_name = f"{module_name}.{child_name}" if len(module_name) > 0 else child_name + attach_align_device_hook_on_blocks( + child, + execution_device=execution_device, + offload=offload, + weights_map=weights_map, + offload_buffers=offload_buffers, + module_name=child_name, + preload_module_classes=preload_module_classes, + skip_keys=skip_keys, + tied_params_map=tied_params_map, + ) + + +class CpuOffload(ModelHook): + """ + Offloads a model on the CPU until its forward pass is called. The model will not be offloaded back to the CPU after + the forward, the user needs to call the `init_hook` method again for this. + + Args: + execution_device(`str`, `int` or `torch.device`, *optional*): + The device on which the model should be executed. Will default to the MPS device if it's available, then + GPU 0 if there is a GPU, and finally to the CPU. + prev_module_hook (`UserCpuOffloadHook`, *optional*): + The hook sent back by [`cpu_offload_with_hook`] for a previous model in the pipeline you are running. If + passed, its offload method will be called just before the forward of the model to which this hook is + attached. + """ + + def __init__( + self, + execution_device: Optional[Union[str, int, torch.device]] = None, + prev_module_hook: Optional["UserCpuOffloadHook"] = None, + ): + self.prev_module_hook = prev_module_hook + + self.execution_device = execution_device if execution_device is not None else PartialState().default_device + + def init_hook(self, module): + return module.to("cpu") + + @_compiler_disable + def pre_forward(self, module, *args, **kwargs): + if self.prev_module_hook is not None and isinstance(self.prev_module_hook, UserCpuOffloadHook): + prev_module = self.prev_module_hook.model + prev_device = next(prev_module.parameters()).device + + # Only offload the previous module if it is not already on CPU. + if prev_device != torch.device("cpu"): + self.prev_module_hook.offload() + clear_device_cache() + + # If the current device is already the self.execution_device, we can skip the transfer. + current_device = next(module.parameters()).device + if current_device == self.execution_device: + return args, kwargs + + module.to(self.execution_device) + return send_to_device(args, self.execution_device), send_to_device(kwargs, self.execution_device) + + +class UserCpuOffloadHook: + """ + A simple hook grouping a model and a `ModelHook`, which provides easy APIs for to call the init method of the hook + or remove it entirely. + """ + + def __init__(self, model, hook): + self.model = model + self.hook = hook + + def offload(self): + self.hook.init_hook(self.model) + + def remove(self): + remove_hook_from_module(self.model) + + +class LayerwiseCastingHook(ModelHook): + r""" + A hook that casts the weights of a module to a high precision dtype for computation, and to a low precision dtype + for storage. This process may lead to quality loss in the output, but can significantly reduce the memory + footprint. + """ + + _is_stateful = False + + def __init__(self, storage_dtype: torch.dtype, compute_dtype: torch.dtype, non_blocking: bool) -> None: + self.storage_dtype = storage_dtype + self.compute_dtype = compute_dtype + self.non_blocking = non_blocking + + def init_hook(self, module: torch.nn.Module): + module.to(dtype=self.storage_dtype, non_blocking=self.non_blocking) + return module + + @_compiler_disable + def pre_forward(self, module: torch.nn.Module, *args, **kwargs): + module.to(dtype=self.compute_dtype, non_blocking=self.non_blocking) + return args, kwargs + + @_compiler_disable + def post_forward(self, module: torch.nn.Module, output): + module.to(dtype=self.storage_dtype, non_blocking=self.non_blocking) + return output diff --git a/kbidu_env/Lib/site-packages/accelerate/inference.py b/kbidu_env/Lib/site-packages/accelerate/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..35c99399baba62f77bbaac8adf4e2eacdbabf779 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/inference.py @@ -0,0 +1,186 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from types import MethodType +from typing import Any, Optional, Union + +from .state import PartialState +from .utils import ( + calculate_maximum_sizes, + convert_bytes, + copy_tensor_to_devices, + ignorant_find_batch_size, + infer_auto_device_map, + is_pippy_available, + pad_input_tensors, + send_to_device, +) + + +def generate_device_map( + model, num_processes: int = 1, no_split_module_classes=None, max_memory: Optional[dict] = None +): + """ + Calculates the device map for `model` with an offset for PiPPy + """ + if num_processes == 1: + return infer_auto_device_map(model, no_split_module_classes=no_split_module_classes, clean_result=False) + if max_memory is None: + model_size, shared = calculate_maximum_sizes(model) + + # Split into `n` chunks for each GPU + memory = (model_size + shared[0]) / num_processes + memory = convert_bytes(memory) + value, ending = memory.split(" ") + + # Add a chunk to deal with potential extra shared memory instances + memory = math.ceil(float(value)) * 1.1 + memory = f"{memory} {ending}" + max_memory = {i: memory for i in range(num_processes)} + device_map = infer_auto_device_map( + model, + max_memory=max_memory, + no_split_module_classes=no_split_module_classes, + clean_result=False, + ) + return device_map + + +def find_pippy_batch_size(args, kwargs): + found_batch_size = None + if args is not None: + for arg in args: + found_batch_size = ignorant_find_batch_size(arg) + if found_batch_size is not None: + break + if kwargs is not None and found_batch_size is None: + for kwarg in kwargs.values(): + found_batch_size = ignorant_find_batch_size(kwarg) + if found_batch_size is not None: + break + return found_batch_size + + +def build_pipeline(model, split_points, args, kwargs, num_chunks): + """ + Attaches the split points to the model based on `self.device_map` and generates a `PipelineStage`. Requires passing + in needed `args` and `kwargs` as the model needs on the CPU. + + Users can pass in custom `num_chunks` as an optional hyper-parameter. By default will use + `AcceleratorState.num_processes` + """ + # Note: We import here to reduce import time from general modules, and isolate outside dependencies + from torch.distributed.pipelining import ScheduleGPipe, SplitPoint, pipeline + + # We need to annotate the split points in the model for PiPPy + state = PartialState() + split_spec = {split_point: SplitPoint.BEGINNING for split_point in split_points} + pipe = pipeline( + model, + mb_args=args, + mb_kwargs=kwargs, + split_spec=split_spec, + ) + stage = pipe.build_stage(state.local_process_index, device=state.device) + schedule = ScheduleGPipe(stage, num_chunks) + + return schedule + + +def pippy_forward(forward, num_chunks, gather_output, *args, **kwargs): + state = PartialState() + output = None + + if state.num_processes == 1: + output = forward(*args, **kwargs) + elif state.is_local_main_process: + found_batch_size = find_pippy_batch_size(args, kwargs) + if found_batch_size is None: + raise ValueError("Could not find batch size from args or kwargs") + else: + if found_batch_size != num_chunks: + args = pad_input_tensors(args, found_batch_size, num_chunks) + kwargs = pad_input_tensors(kwargs, found_batch_size, num_chunks) + forward(*args, **kwargs) + elif state.is_last_process: + output = forward() + else: + forward() + if gather_output: + # Each node will get a copy of the full output which is only on the last GPU + output = copy_tensor_to_devices(output) + return output + + +def prepare_pippy( + model, + split_points: Optional[Union[str, list[str]]] = "auto", + no_split_module_classes: Optional[list[str]] = None, + example_args: Optional[tuple[Any]] = (), + example_kwargs: Optional[dict[str, Any]] = None, + num_chunks: Optional[int] = None, + gather_output: Optional[bool] = False, +): + """ + Wraps `model` for pipeline parallel inference. + + Args: + model (`torch.nn.Module`): + A model we want to split for pipeline-parallel inference + split_points (`str` or `List[str]`, defaults to 'auto'): + How to generate the split points and chunk the model across each GPU. 'auto' will find the best balanced + split given any model. Should be a list of layer names in the model to split by otherwise. + no_split_module_classes (`List[str]`): + A list of class names for layers we don't want to be split. + example_args (tuple of model inputs): + The expected inputs for the model that uses order-based inputs for a *single process*. Recommended to use + this method if possible. + example_kwargs (dict of model inputs) + The expected inputs for the model that uses dictionary-based inputs for a *single process*. This is a + *highly* limiting structure that requires the same keys be present at *all* inference calls. Not + recommended unless the prior condition is true for all cases. + num_chunks (`int`, defaults to the number of available GPUs): + The number of different stages the Pipeline will have. By default it will assign one chunk per GPU, but + this can be tuned and played with. In general one should have num_chunks >= num_gpus. + gather_output (`bool`, defaults to `False`): + If `True`, the output from the last GPU (which holds the true outputs) is sent across to all GPUs. + """ + if not is_pippy_available(): + raise ImportError("Using `torch.distributed.pipelining` requires PyTorch 2.4.0 or later.") + state = PartialState() + example_args = send_to_device(example_args, "cpu") + example_kwargs = send_to_device(example_kwargs, "cpu") + if num_chunks is None: + num_chunks = state.num_processes + if split_points == "auto": + device_map = generate_device_map(model, num_chunks, no_split_module_classes=no_split_module_classes) + split_points = [] + for i in range(1, num_chunks): + split_points.append(next(k for k, v in device_map.items() if v == i)) + model.hf_split_points = split_points + stage = build_pipeline(model, split_points, example_args, example_kwargs, num_chunks) + model._original_forward = model.forward + model._original_call = model.__call__ + model.pippy_stage = stage + model.hf_split_points = split_points + + def forward(*args, **kwargs): + return pippy_forward(stage.step, num_chunks, gather_output, *args, **kwargs) + + # To act like a decorator so that it can be popped when doing `extract_model_from_parallel` + # Note: creates an infinite recursion loop with `generate` + model_forward = MethodType(forward, model) + forward.__wrapped__ = model_forward + model.forward = forward + return model diff --git a/kbidu_env/Lib/site-packages/accelerate/launchers.py b/kbidu_env/Lib/site-packages/accelerate/launchers.py new file mode 100644 index 0000000000000000000000000000000000000000..ac76fc51efba7bfe117c00a2cfa210b294936e0b --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/launchers.py @@ -0,0 +1,322 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import tempfile +from dataclasses import fields + +import torch + +from .state import AcceleratorState, PartialState +from .utils import ( + PrecisionType, + PrepareForLaunch, + are_libraries_initialized, + check_cuda_p2p_ib_support, + get_current_device_type, + get_gpu_info, + is_mps_available, + is_rocm_available, + is_torch_version, + patch_environment, +) +from .utils.constants import ELASTIC_LOG_LINE_PREFIX_TEMPLATE_PYTORCH_VERSION + + +def test_launch(): + "Verify a `PartialState` can be initialized." + _ = PartialState() + + +def notebook_launcher( + function, + args=(), + num_processes=None, + mixed_precision="no", + use_port="29500", + master_addr="127.0.0.1", + node_rank=0, + num_nodes=1, + rdzv_backend="static", + rdzv_endpoint="", + rdzv_conf=None, + rdzv_id="none", + max_restarts=0, + monitor_interval=0.1, + log_line_prefix_template=None, +): + """ + Launches a training function, using several processes or multiple nodes if it's possible in the current environment + (TPU with multiple cores for instance). + + + + To use this function absolutely zero calls to a device must be made in the notebook session before calling. If any + have been made, you will need to restart the notebook and make sure no cells use any device capability. + + Setting `ACCELERATE_DEBUG_MODE="1"` in your environment will run a test before truly launching to ensure that none + of those calls have been made. + + + + Args: + function (`Callable`): + The training function to execute. If it accepts arguments, the first argument should be the index of the + process run. + args (`Tuple`): + Tuple of arguments to pass to the function (it will receive `*args`). + num_processes (`int`, *optional*): + The number of processes to use for training. Will default to 8 in Colab/Kaggle if a TPU is available, to + the number of devices available otherwise. + mixed_precision (`str`, *optional*, defaults to `"no"`): + If `fp16` or `bf16`, will use mixed precision training on multi-device. + use_port (`str`, *optional*, defaults to `"29500"`): + The port to use to communicate between processes when launching a multi-device training. + master_addr (`str`, *optional*, defaults to `"127.0.0.1"`): + The address to use for communication between processes. + node_rank (`int`, *optional*, defaults to 0): + The rank of the current node. + num_nodes (`int`, *optional*, defaults to 1): + The number of nodes to use for training. + rdzv_backend (`str`, *optional*, defaults to `"static"`): + The rendezvous method to use, such as 'static' (the default) or 'c10d' + rdzv_endpoint (`str`, *optional*, defaults to `""`): + The endpoint of the rdzv sync. storage. + rdzv_conf (`Dict`, *optional*, defaults to `None`): + Additional rendezvous configuration. + rdzv_id (`str`, *optional*, defaults to `"none"`): + The unique run id of the job. + max_restarts (`int`, *optional*, defaults to 0): + The maximum amount of restarts that elastic agent will conduct on workers before failure. + monitor_interval (`float`, *optional*, defaults to 0.1): + The interval in seconds that is used by the elastic_agent as a period of monitoring workers. + log_line_prefix_template (`str`, *optional*, defaults to `None`): + The prefix template for elastic launch logging. Available from PyTorch 2.2.0. + + Example: + + ```python + # Assume this is defined in a Jupyter Notebook on an instance with two devices + from accelerate import notebook_launcher + + + def train(*args): + # Your training function here + ... + + + notebook_launcher(train, args=(arg1, arg2), num_processes=2, mixed_precision="fp16") + ``` + """ + # Are we in a google colab or a Kaggle Kernel? + in_colab = False + in_kaggle = False + if any(key.startswith("KAGGLE") for key in os.environ.keys()): + in_kaggle = True + elif "IPython" in sys.modules: + in_colab = "google.colab" in str(sys.modules["IPython"].get_ipython()) + + try: + mixed_precision = PrecisionType(mixed_precision.lower()) + except ValueError: + raise ValueError( + f"Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}." + ) + + if (in_colab or in_kaggle) and ( + (os.environ.get("TPU_NAME", None) is not None) or (os.environ.get("PJRT_DEVICE", "") == "TPU") + ): + # TPU launch + import torch_xla.distributed.xla_multiprocessing as xmp + + if len(AcceleratorState._shared_state) > 0: + raise ValueError( + "To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside " + "your training function. Restart your notebook and make sure no cells initializes an " + "`Accelerator`." + ) + + launcher = PrepareForLaunch(function, distributed_type="XLA") + print("Launching a training on TPU cores.") + xmp.spawn(launcher, args=args, start_method="fork") + elif in_colab and (not torch.cuda.is_available() or get_gpu_info()[1] < 2): + # No need for a distributed launch otherwise as it's either CPU or one GPU. + if torch.cuda.is_available(): + print("Launching training on one GPU.") + else: + print("Launching training on one CPU.") + function(*args) + else: + if num_processes is None: + raise ValueError( + "You have to specify the number of devices you would like to use, add `num_processes=...` to your call." + ) + if node_rank >= num_nodes: + raise ValueError("The node_rank must be less than the number of nodes.") + if num_processes > 1: + # Multi-device launch + from torch.distributed.launcher.api import LaunchConfig, elastic_launch + from torch.multiprocessing import start_processes + from torch.multiprocessing.spawn import ProcessRaisedException + + if len(AcceleratorState._shared_state) > 0: + raise ValueError( + "To launch a multi-device training from your notebook, the `Accelerator` should only be initialized " + "inside your training function. Restart your notebook and make sure no cells initializes an " + "`Accelerator`." + ) + # Check for specific libraries known to initialize device that users constantly use + problematic_imports = are_libraries_initialized("bitsandbytes") + if len(problematic_imports) > 0: + err = ( + "Could not start distributed process. Libraries known to initialize device upon import have been " + "imported already. Please keep these imports inside your training function to try and help with this:" + ) + for lib_name in problematic_imports: + err += f"\n\t* `{lib_name}`" + raise RuntimeError(err) + + patched_env = dict( + nproc=num_processes, + node_rank=node_rank, + world_size=num_nodes * num_processes, + master_addr=master_addr, + master_port=use_port, + mixed_precision=mixed_precision, + ) + + # Check for CUDA P2P and IB issues + if not check_cuda_p2p_ib_support(): + patched_env["nccl_p2p_disable"] = "1" + patched_env["nccl_ib_disable"] = "1" + + # torch.distributed will expect a few environment variable to be here. We set the ones common to each + # process here (the other ones will be set be the launcher). + with patch_environment(**patched_env): + # First dummy launch + # Determine device type without initializing any device (which would break fork) + device_type, distributed_type = get_current_device_type() + # XPU and ROCm require spawn instead of fork (HIP/XPU runtime is initialized in the parent, + # which breaks fork-based subprocesses). + start_method = "spawn" if device_type == "xpu" or is_rocm_available() else "fork" + if os.environ.get("ACCELERATE_DEBUG_MODE", "false").lower() == "true": + launcher = PrepareForLaunch(test_launch, distributed_type=distributed_type) + try: + start_processes(launcher, args=(), nprocs=num_processes, start_method=start_method) + except ProcessRaisedException as e: + err = "An issue was found when verifying a stable environment for the notebook launcher." + if f"Cannot re-initialize {device_type.upper()} in forked subprocess" in e.args[0]: + raise RuntimeError( + f"{err}" + "This likely stems from an outside import causing issues once the `notebook_launcher()` is called. " + "Please review your imports and test them when running the `notebook_launcher()` to identify " + f"which one is problematic and causing {device_type.upper()} to be initialized." + ) from e + else: + raise RuntimeError(f"{err} The following error was raised: {e}") from e + # Now the actual launch + launcher = PrepareForLaunch(function, distributed_type=distributed_type) + print(f"Launching training on {num_processes} {device_type.upper()}s.") + try: + if rdzv_conf is None: + rdzv_conf = {} + if rdzv_backend == "static": + rdzv_conf["rank"] = node_rank + if not rdzv_endpoint: + rdzv_endpoint = f"{master_addr}:{use_port}" + launch_config_kwargs = dict( + min_nodes=num_nodes, + max_nodes=num_nodes, + nproc_per_node=num_processes, + run_id=rdzv_id, + rdzv_endpoint=rdzv_endpoint, + rdzv_backend=rdzv_backend, + rdzv_configs=rdzv_conf, + max_restarts=max_restarts, + monitor_interval=monitor_interval, + start_method=start_method, + ) + if is_torch_version(">=", ELASTIC_LOG_LINE_PREFIX_TEMPLATE_PYTORCH_VERSION): + launch_config_kwargs["log_line_prefix_template"] = log_line_prefix_template + has_numa_options = any(field.name == "numa_options" for field in fields(LaunchConfig)) + if has_numa_options: + from torch.numa.binding import AffinityMode, NumaOptions + + launch_config_kwargs["numa_options"] = NumaOptions(AffinityMode.NODE) + launch_config = LaunchConfig(**launch_config_kwargs) + if has_numa_options: + launch_config.numa_options = None + elastic_launch(config=launch_config, entrypoint=function)(*args) + except ProcessRaisedException as e: + if f"Cannot re-initialize {device_type.upper()} in forked subprocess" in e.args[0]: + raise RuntimeError( + f"{device_type.upper()} has been initialized before the `notebook_launcher` could create a forked subprocess. " + "This likely stems from an outside import causing issues once the `notebook_launcher()` is called. " + "Please review your imports and test them when running the `notebook_launcher()` to identify " + f"which one is problematic and causing {device_type.upper()} to be initialized." + ) from e + else: + raise RuntimeError(f"An issue was found when launching the training: {e}") from e + + else: + # No need for a distributed launch otherwise as it's either CPU, GPU, XPU or MPS. + if is_mps_available(): + os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" + print("Launching training on MPS.") + elif torch.cuda.is_available(): + print("Launching training on one GPU.") + elif torch.xpu.is_available(): + print("Launching training on one XPU.") + else: + print("Launching training on CPU.") + function(*args) + + +def debug_launcher(function, args=(), num_processes=2): + """ + Launches a training function using several processes on CPU for debugging purposes. + + + + This function is provided for internal testing and debugging, but it's not intended for real trainings. It will + only use the CPU. + + + + Args: + function (`Callable`): + The training function to execute. + args (`Tuple`): + Tuple of arguments to pass to the function (it will receive `*args`). + num_processes (`int`, *optional*, defaults to 2): + The number of processes to use for training. + """ + from torch.multiprocessing import start_processes + + with tempfile.NamedTemporaryFile() as tmp_file: + # torch.distributed will expect a few environment variable to be here. We set the ones common to each + # process here (the other ones will be set be the launcher). + # gloo's default interface selection (hostname-based) is flaky on CI runners, pin it to loopback + with patch_environment( + world_size=num_processes, + master_addr="127.0.0.1", + master_port="29500", + accelerate_mixed_precision="no", + accelerate_debug_rdv_file=tmp_file.name, + accelerate_use_cpu="yes", + gloo_socket_ifname="lo", + ): + launcher = PrepareForLaunch(function, debug=True) + start_processes(launcher, args=args, nprocs=num_processes, start_method="fork") diff --git a/kbidu_env/Lib/site-packages/accelerate/local_sgd.py b/kbidu_env/Lib/site-packages/accelerate/local_sgd.py new file mode 100644 index 0000000000000000000000000000000000000000..0ebac2b2c0b89a781c35fc8d9bc3971ed58d50fd --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/local_sgd.py @@ -0,0 +1,107 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from accelerate import Accelerator, DistributedType + + +class LocalSGD: + """ + A helper class to support local SGD on top of Accelerator. It simply runs a given number of updates independently + on each device, and averages model weights every K synchronization step. + + It should be used only in the multi-GPU (or multi-CPU) setup without extensions such as DeepSpeed. In particular, + this is a simple implementation that cannot support scenarios such as model parallelism. + + + Although we are not aware of the true origins of this simple approach, the idea of local SGD is quite old and goes + back to at least: + + Zhang, J., De Sa, C., Mitliagkas, I., & Ré, C. (2016). [Parallel SGD: When does averaging help?. arXiv preprint + arXiv:1606.07365.](https://huggingface.co/papers/1606.07365) + + We credit the term Local SGD to the following paper (but there might be earlier references we are not aware of). + + Stich, Sebastian Urban. ["Local SGD Converges Fast and Communicates Little." ICLR 2019-International Conference on + Learning Representations. No. CONF. 2019.](https://huggingface.co/papers/1805.09767) + + """ + + def __enter__(self): + if self.enabled: + self.model_sync_obj = self.model.no_sync() + self.model_sync_obj.__enter__() + + return self + + def __exit__(self, type, value, tb): + if self.enabled: + # Average all models on exit + self._sync_and_avg_model_params() + self.model_sync_obj.__exit__(type, value, tb) + + def __init__(self, accelerator: Accelerator, model: torch.nn.Module, local_sgd_steps: int, enabled: bool = True): + """ + Constructor. + + Args: + model (`torch.nn.Module): + The model whose parameters we need to average. + accelerator (`Accelerator`): + Accelerator object. + local_sgd_steps (`int`): + A number of local SGD steps (before model parameters are synchronized). + enabled (`bool): + Local SGD is disabled if this parameter set to `False`. + """ + if accelerator.distributed_type not in [ + DistributedType.NO, + DistributedType.MULTI_CPU, + DistributedType.MULTI_GPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NPU, + DistributedType.MULTI_NEURON, + ]: + raise NotImplementedError("LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)") + self.enabled = enabled and accelerator.distributed_type != DistributedType.NO + self.num_steps = 0 + if self.enabled: + self.accelerator = accelerator + self.model = model + self.local_sgd_steps = local_sgd_steps + + def step(self): + """ + This function makes a "step" and synchronizes model parameters if necessary. + """ + self.num_steps += 1 + if not self.enabled: + return + + if self.num_steps % self.local_sgd_steps == 0: + self._sync_and_avg_model_params() + + def _sync_and_avg_model_params(self): + """ + Synchronize + Average model parameters across all GPUs + """ + + self.accelerator.wait_for_everyone() + with self.accelerator.autocast(): + for param in self.model.parameters(): + param.data = self.accelerator.reduce(param.data, reduction="mean") diff --git a/kbidu_env/Lib/site-packages/accelerate/logging.py b/kbidu_env/Lib/site-packages/accelerate/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..9132f8cbf4b5c06d48bec4b941b6cd96c566045d --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/logging.py @@ -0,0 +1,133 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import functools +import logging +import os + +from .state import PartialState + + +class MultiProcessAdapter(logging.LoggerAdapter): + """ + An adapter to assist with logging in multiprocess. + + `log` takes in an additional `main_process_only` kwarg, which dictates whether it should be called on all processes + or only the main executed one. Default is `main_process_only=True`. + + Does not require an `Accelerator` object to be created first. + """ + + @staticmethod + def _should_log(main_process_only): + "Check if log should be performed" + state = PartialState() + return not main_process_only or (main_process_only and state.is_main_process) + + def process(self, msg, kwargs): + msg, kwargs = super().process(msg, kwargs) + + # set `stacklevel` to exclude ourself in `Logger.findCaller()` while respecting user's choice + kwargs.setdefault("stacklevel", 2) + + state = PartialState() + msg = f"[RANK {state.process_index}] {msg}" + return msg, kwargs + + def log(self, level, msg, *args, **kwargs): + """ + Delegates logger call after checking if we should log. + + Accepts a new kwarg of `main_process_only`, which will dictate whether it will be logged across all processes + or only the main executed one. Default is `True` if not passed + + Also accepts "in_order", which if `True` makes the processes log one by one, in order. This is much easier to + read, but comes at the cost of sometimes needing to wait for the other processes. Default is `False` to not + break with the previous behavior. + + `main_process_only` is ignored if `in_order` is passed. + """ + if PartialState._shared_state == {}: + raise RuntimeError( + "You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility." + ) + main_process_only = kwargs.pop("main_process_only", True) + in_order = kwargs.pop("in_order", False) + + if self.isEnabledFor(level): + msg, kwargs = self.process(msg, kwargs) + if not in_order and self._should_log(main_process_only): + self.logger.log(level, msg, *args, **kwargs) + + elif in_order: + state = PartialState() + for i in range(state.num_processes): + if i == state.process_index: + self.logger.log(level, msg, *args, **kwargs) + state.wait_for_everyone() + + @functools.lru_cache(None) + def warning_once(self, *args, **kwargs): + """ + This method is identical to `logger.warning()`, but will emit the warning with the same message only once + + Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the + cache. The assumption here is that all warning messages are unique across the code. If they aren't then need to + switch to another type of cache that includes the caller frame information in the hashing function. + """ + self.warning(*args, **kwargs) + + +def get_logger(name: str, log_level: str | None = None): + """ + Returns a `logging.Logger` for `name` that can handle multiprocessing. + + If a log should be called on all processes, pass `main_process_only=False` If a log should be called on all + processes and in order, also pass `in_order=True` + + Args: + name (`str`): + The name for the logger, such as `__file__` + log_level (`str`, *optional*): + The log level to use. If not passed, will default to the `LOG_LEVEL` environment variable, or `INFO` if not + + Example: + + ```python + >>> from accelerate.logging import get_logger + >>> from accelerate import Accelerator + + >>> logger = get_logger(__name__) + + >>> accelerator = Accelerator() + >>> logger.info("My log", main_process_only=False) + >>> logger.debug("My log", main_process_only=True) + + >>> logger = get_logger(__name__, log_level="DEBUG") + >>> logger.info("My log") + >>> logger.debug("My second log") + + >>> array = ["a", "b", "c", "d"] + >>> letter_at_rank = array[accelerator.process_index] + >>> logger.info(letter_at_rank, in_order=True) + ``` + """ + if log_level is None: + log_level = os.environ.get("ACCELERATE_LOG_LEVEL", None) + logger = logging.getLogger(name) + if log_level is not None: + logger.setLevel(log_level.upper()) + logger.root.setLevel(log_level.upper()) + return MultiProcessAdapter(logger, {}) diff --git a/kbidu_env/Lib/site-packages/accelerate/memory_utils.py b/kbidu_env/Lib/site-packages/accelerate/memory_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fa2e2c8b9d7d0064c3e5e282737a7ad6919bde29 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/memory_utils.py @@ -0,0 +1,22 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings + + +warnings.warn( + "memory_utils has been reorganized to utils.memory. Import `find_executable_batchsize` from the main `__init__`: " + "`from accelerate import find_executable_batch_size` to avoid this warning.", + FutureWarning, +) diff --git a/kbidu_env/Lib/site-packages/accelerate/optimizer.py b/kbidu_env/Lib/site-packages/accelerate/optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..c1f8faa15436fb5d1d02ae4d691a4fcac44cd236 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/optimizer.py @@ -0,0 +1,213 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect + +import torch + +from .state import AcceleratorState, GradientState +from .utils import DistributedType, honor_type, is_lomo_available, is_torch_xla_available + + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + import torch_xla.runtime as xr + + +def move_to_device(state, device): + if isinstance(state, (list, tuple)): + return honor_type(state, (move_to_device(t, device) for t in state)) + elif isinstance(state, dict): + return type(state)({k: move_to_device(v, device) for k, v in state.items()}) + elif isinstance(state, torch.Tensor): + return state.to(device) + return state + + +class AcceleratedOptimizer(torch.optim.Optimizer): + """ + Internal wrapper around a torch optimizer. + + Conditionally will perform `step` and `zero_grad` if gradients should be synchronized when performing gradient + accumulation. + + Args: + optimizer (`torch.optim.optimizer.Optimizer`): + The optimizer to wrap. + device_placement (`bool`, *optional*, defaults to `True`): + Whether or not the optimizer should handle device placement. If so, it will place the state dictionary of + `optimizer` on the right device. + scaler (`torch.amp.GradScaler` or `torch.cuda.amp.GradScaler`, *optional*): + The scaler to use in the step function if training with mixed precision. + """ + + def __init__(self, optimizer, device_placement=True, scaler=None): + self.optimizer = optimizer + self.scaler = scaler + self.accelerator_state = AcceleratorState() + self.gradient_state = GradientState() + self.device_placement = device_placement + self._is_overflow = False + + if self.scaler is not None: + self._accelerate_step_called = False + self._optimizer_original_step_method = self.optimizer.step + self._optimizer_patched_step_method = patch_optimizer_step(self, self.optimizer.step) + + # Handle device placement + if device_placement: + state_dict = self.optimizer.state_dict() + if self.accelerator_state.distributed_type == DistributedType.XLA: + xm.send_cpu_data_to_device(state_dict, self.accelerator_state.device) + else: + state_dict = move_to_device(state_dict, self.accelerator_state.device) + self.optimizer.load_state_dict(state_dict) + + @property + def state(self): + return self.optimizer.state + + @state.setter + def state(self, state): + self.optimizer.state = state + + @property + def param_groups(self): + return self.optimizer.param_groups + + @param_groups.setter + def param_groups(self, param_groups): + self.optimizer.param_groups = param_groups + + @property + def defaults(self): + return self.optimizer.defaults + + @defaults.setter + def defaults(self, defaults): + self.optimizer.defaults = defaults + + def add_param_group(self, param_group): + self.optimizer.add_param_group(param_group) + + def load_state_dict(self, state_dict): + if self.accelerator_state.distributed_type == DistributedType.XLA and self.device_placement: + xm.send_cpu_data_to_device(state_dict, self.accelerator_state.device) + self.optimizer.load_state_dict(state_dict) + + def state_dict(self): + return self.optimizer.state_dict() + + def zero_grad(self, set_to_none=None): + if self.gradient_state.sync_gradients: + accept_arg = "set_to_none" in inspect.signature(self.optimizer.zero_grad).parameters + if accept_arg: + if set_to_none is None: + set_to_none = True + self.optimizer.zero_grad(set_to_none=set_to_none) + else: + if set_to_none is not None: + raise ValueError("`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.") + self.optimizer.zero_grad() + + def train(self): + """ + Sets the optimizer to "train" mode. Useful for optimizers like `schedule_free` + """ + if hasattr(self.optimizer, "train") and callable(self.optimizer.train): + self.optimizer.train() + elif ( + hasattr(self.optimizer, "optimizer") + and hasattr(self.optimizer.optimizer, "train") + and callable(self.optimizer.optimizer.train) + ): + # the deepspeed optimizer further wraps the optimizer + self.optimizer.optimizer.train() + + def eval(self): + """ + Sets the optimizer to "eval" mode. Useful for optimizers like `schedule_free` + """ + if hasattr(self.optimizer, "eval") and callable(self.optimizer.eval): + self.optimizer.eval() + + def step(self, closure=None): + if is_lomo_available(): + from lomo_optim import AdaLomo, Lomo + + if ( + not self.gradient_state.is_xla_gradients_synced + and self.accelerator_state.distributed_type == DistributedType.XLA + ): + gradients = xm._fetch_gradients(self.optimizer) + xm.all_reduce("sum", gradients, scale=1.0 / xr.world_size()) + self.gradient_state.is_xla_gradients_synced = True + + if is_lomo_available(): + # `step` should be a no-op for LOMO optimizers. + if isinstance(self.optimizer, (Lomo, AdaLomo)): + return + + if self.gradient_state.sync_gradients: + if self.scaler is not None: + self.optimizer.step = self._optimizer_patched_step_method + + self.scaler.step(self.optimizer, closure) + self.scaler.update() + + if not self._accelerate_step_called: + # If the optimizer step was skipped, gradient overflow was detected. + self._is_overflow = True + else: + self._is_overflow = False + # Reset the step method to the original one + self.optimizer.step = self._optimizer_original_step_method + # Reset the indicator + self._accelerate_step_called = False + else: + self.optimizer.step(closure) + if self.accelerator_state.distributed_type == DistributedType.XLA: + self.gradient_state.is_xla_gradients_synced = False + + def _switch_parameters(self, parameters_map): + for param_group in self.optimizer.param_groups: + param_group["params"] = [parameters_map.get(p, p) for p in param_group["params"]] + + @property + def step_was_skipped(self): + """Whether or not the optimizer step was skipped.""" + return self._is_overflow + + def __getstate__(self): + _ignored_keys = [ + "_accelerate_step_called", + "_optimizer_original_step_method", + "_optimizer_patched_step_method", + ] + return {k: v for k, v in self.__dict__.items() if k not in _ignored_keys} + + def __setstate__(self, state): + self.__dict__.update(state) + if self.scaler is not None: + self._accelerate_step_called = False + self._optimizer_original_step_method = self.optimizer.step + self._optimizer_patched_step_method = patch_optimizer_step(self, self.optimizer.step) + + +def patch_optimizer_step(accelerated_optimizer: AcceleratedOptimizer, method): + def patched_step(*args, **kwargs): + accelerated_optimizer._accelerate_step_called = True + return method(*args, **kwargs) + + return patched_step diff --git a/kbidu_env/Lib/site-packages/accelerate/parallelism_config.py b/kbidu_env/Lib/site-packages/accelerate/parallelism_config.py new file mode 100644 index 0000000000000000000000000000000000000000..22d15d3678cdee65347dc914b019ed5a26a74d28 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/parallelism_config.py @@ -0,0 +1,398 @@ +# +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import warnings +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, Optional, Union + +from accelerate.utils.dataclasses import ( + DeepSpeedSequenceParallelConfig, + DistributedType, + TorchContextParallelConfig, + TorchTensorParallelConfig, +) +from accelerate.utils.versions import is_torch_version + + +if TYPE_CHECKING: + from accelerate import Accelerator + + +@dataclass +class ParallelismConfig: + """ + A dataclass to configure parallelisms applied to the model. Inspired by torchtitan's `ParallelDims` + https://github.com/pytorch/torchtitan/blob/main/torchtitan/distributed/parallel_dims.py + + Args: + dp_replicate_size (`int`, defaults to `1`): + The size of the data parallel group. If `dp_replicate_size` is set to 1, the data parallel replication + group will not be used. + dp_shard_size (`int`, defaults to `1`): + The size of the model shard group. If `dp_replicate_size > 1` and `tp_size > 1`, `dp_shard_size` must also + be greater than 1, as composing DDP + TP is currently not supported. + tp_size (`int`, defaults to `1`): + The size of the tensor parallel group. If `tp_size` is set to `1`, the tensor parallel group will not be + used. + tp_handler (`~utils.TorchTensorParallelConfig`, defaults to `None`): + The handler for the tensor parallel group. + cp_size (`int`, defaults to `1`): + The size of the context parallel group. Currently not supported, but reserved for future use and enabled + for downstream libraries. + cp_backend (`str`, defaults to `torch`): + Which CP backend to use: `torch` (FSDP2) + sp_size (`int`, defaults to `1`): + The size of the sequence parallel group. + sp_backend (`str`, defaults to `deepspeed`): + Which SP backend to use:`deepspeed` (ALST/Ulysses) + + You may obtain different distributed data parallel paradigms by configuring `dp_replicate_size` and `dp_shard_size` + together: + - `dp_replicate_size == 1` and `dp_shard_size > 1`, we obtain Fully Sharded Data Parallel (FSDP). + - `dp_replicate_size > 1` and `dp_shard_size > 1`, we obtain Hybrid Sharded Data Parallel (HSDP). + - `dp_replicate_size > 1` and `dp_shard_size == 1` is an invalid configuration, to use pure DP, use + `DistributedDataParallelKwargs` instead. + + """ + + dp_replicate_size: Optional[int] = None + dp_shard_size: Optional[int] = None + tp_size: Optional[int] = None + cp_size: Optional[int] = None + cp_backend: Literal["torch"] = None + sp_size: Optional[int] = None + sp_backend: Literal["deepspeed"] = None + + # we use Union because we might support other x parallel plugins (i.e. deepspeed, etc) + tp_handler: Union[None, TorchTensorParallelConfig] = None + cp_handler: Union[None, TorchContextParallelConfig] = None + sp_handler: Union[None, DeepSpeedSequenceParallelConfig] = None + + device_mesh = None + + def __repr__(self): + return ( + "ParallelismConfig(\n " + f"\tdp_replicate_size={self.dp_replicate_size},\n" + f"\tdp_shard_size={self.dp_shard_size},\n" + f"\ttp_size={self.tp_size},\n" + f"\tcp_size={self.cp_size},\n" + f"\tcp_backend={self.cp_backend},\n" + f"\tsp_size={self.sp_size},\n" + f"\tsp_backend={self.sp_backend},\n" + f"\ttotal_size={self.total_size}\n" + f"\ttp_handler={self.tp_handler},\n" + f"\tcp_handler={self.cp_handler})\n" + ) + + def to_json(self): + import copy + + _non_serializable_fields = ["device_mesh"] + + copy.deepcopy( + { + k: copy.deepcopy(v.__dict__) if hasattr(v, "__dict__") else v + for k, v in self.__dict__.items() + if k not in _non_serializable_fields + } + ) + + @property + def dp_dim_names(self): + """Names of enabled dimensions across which data parallelism is applied.""" + dims = [] + if self.dp_replicate_enabled: + dims += ["dp_replicate"] + if self.dp_shard_enabled: + dims += ["dp_shard"] + return dims + + @property + def non_dp_dim_names(self): + """Names of enabled dimensions which will receive the same batch (non-data parallel dimensions).""" + dims = [] + if self.tp_enabled: + dims += ["tp"] + if self.cp_enabled: + dims += ["cp"] + if self.sp_enabled: + dims += ["sp"] + return dims + + @property + def dp_shard_cp_dim_names(self): + """Names of enabled dimensions which will be flattened into a joint mesh across which is model sharded in FSDP.""" + dims = [] + if self.dp_shard_enabled: + dims += ["dp_shard"] + if self.cp_enabled: + dims += ["cp"] + return dims + + @property + def dp_cp_dim_names(self): + """Names of enabled dimensions across which loss should be averaged""" + dims = [] + if self.dp_replicate_enabled: + dims += ["dp_replicate"] + if self.dp_shard_enabled: + dims += ["dp_shard"] + if self.cp_enabled: + dims += ["cp"] + return dims + + @property + def fsdp_dim_names(self): + """Names of enabled dimensions across which FSDP is applied, including data parallel replication.""" + dims = [] + if self.dp_replicate_enabled: + dims += ["dp_replicate"] + dims += ["dp_shard_cp"] + return dims + + @property + def total_size(self): + """The total size of the parallelism configuration, which is the product of all sizes.""" + return self.dp_replicate_size * self.dp_shard_size * self.tp_size * self.cp_size * self.sp_size + + @property + def non_data_parallel_size(self): + """The size of the non-data parallel dimensions, which is the product of tensor and context parallel sizes.""" + return self.tp_size * self.cp_size * self.sp_size + + @property + def data_parallel_size(self): + """The size of the data parallel dimensions, which is the product of data parallel replication and""" + return self.dp_replicate_size * self.dp_shard_size + + @property + def dp_replicate_enabled(self): + """True if data parallel replication is enabled, i.e. `dp_replicate_size > 1`.""" + return self.dp_replicate_size > 1 + + @property + def dp_shard_enabled(self): + """True if data parallel sharding is enabled, i.e. `dp_shard_size > 1`.""" + return self.dp_shard_size > 1 + + @property + def tp_enabled(self): + """True if tensor parallelism is enabled, i.e. `tp_size > 1`.""" + return self.tp_size > 1 + + @property + def cp_enabled(self): + """True if context parallelism is enabled, i.e. `cp_size > 1`.""" + return self.cp_size > 1 + + @property + def sp_enabled(self): + """True if context parallelism is enabled, i.e. `sp_size > 1`.""" + return self.sp_size > 1 + + @property + def active_mesh_dims(self): + """Names of all active mesh dimensions.""" + return self.dp_dim_names + self.non_dp_dim_names + + def build_device_mesh(self, device_type: str): + """Builds a device mesh for the given device type based on the parallelism configuration. + This method will also create required joint meshes (e.g. `dp_shard_cp`, `dp_cp`, `dp`). + + Args: + device_type (`str`): The type of device for which to build the mesh, e + """ + # Skip mesh creation for DeepSpeed SP - DeepSpeed handles its own SP groups + # Only skip when SP is actually enabled (sp_size > 1), otherwise user might still want TP/CP/FSDP + if self.sp_backend == "deepspeed" and self.sp_size > 1: + return None + + if is_torch_version(">=", "2.2.0"): + from torch.distributed.device_mesh import init_device_mesh + else: + raise RuntimeError("Building a device_mesh requires to have torch>=2.2.0") + + mesh = self._get_mesh() + if len(mesh) == 0: + return None + mesh_dim_names, mesh_shape = mesh + device_mesh = init_device_mesh( + device_type, + mesh_shape, + mesh_dim_names=mesh_dim_names, + ) + if self.dp_dim_names: + device_mesh[self.dp_dim_names]._flatten("dp") + if self.dp_shard_cp_dim_names: + device_mesh[self.dp_shard_cp_dim_names]._flatten("dp_shard_cp") + if self.dp_cp_dim_names: + device_mesh[self.dp_cp_dim_names]._flatten("dp_cp") + + return device_mesh + + def get_device_mesh(self, device_type: Optional[str] = None): + if self.device_mesh is None: + if device_type is not None: + self.device_mesh = self.build_device_mesh(device_type) + else: + raise ValueError("You need to pass a device_type e.g cuda to build the device mesh") + else: + if device_type is not None: + if self.device_mesh.device_type != device_type: + raise ValueError( + f"The device_mesh is already created with device type {self.device_mesh.device_type}. However, you are trying to get a device mesh with device_type {device_type}. Please check if you correctly initialized your device_mesh" + ) + return self.device_mesh + + def _get_mesh(self) -> tuple[tuple[int, ...], tuple[str, ...]]: + """Generate mesh shape and dimension names for torch.distributed.init_device_mesh().""" + + # Build mesh dimensions dictionary + mesh_dims = {parallelism: self._sizes[parallelism] for parallelism in self.active_mesh_dims} + + # Apply canonical ordering + mesh_order = ["dp_replicate", "dp_shard", "cp", "sp", "tp"] + sorted_items = sorted( + mesh_dims.items(), + key=lambda x: (mesh_order.index(x[0])), + ) + return tuple(zip(*sorted_items)) + + def __post_init__(self): + # Basic size validation + if self.dp_replicate_size is None: + self.dp_replicate_size = int(os.environ.get("PARALLELISM_CONFIG_DP_REPLICATE_SIZE", "1")) + if self.dp_shard_size is None: + self.dp_shard_size = int(os.environ.get("PARALLELISM_CONFIG_DP_SHARD_SIZE", "1")) + if self.tp_size is None: + self.tp_size = int(os.environ.get("PARALLELISM_CONFIG_TP_SIZE", "1")) + if self.cp_size is None: + self.cp_size = int(os.environ.get("PARALLELISM_CONFIG_CP_SIZE", "1")) + if self.cp_backend is None: + self.cp_backend = os.environ.get("PARALLELISM_CONFIG_CP_BACKEND", "torch") + if self.sp_size is None: + self.sp_size = int(os.environ.get("PARALLELISM_CONFIG_SP_SIZE", "1")) + if self.sp_backend is None: + self.sp_backend = os.environ.get("PARALLELISM_CONFIG_SP_BACKEND", "deepspeed") + + if self.tp_size > 1: + if self.tp_handler is None: + self.tp_handler = TorchTensorParallelConfig() + + if self.cp_size > 1: + if self.cp_handler is None: + self.cp_handler = TorchContextParallelConfig() + else: + cp_backends_config_map = dict( + torch=TorchContextParallelConfig, + ) + if not isinstance(self.cp_handler, cp_backends_config_map[self.cp_backend]): + raise ValueError( + f"ParallelismConfig's cp_backend={self.cp_backend} requires {cp_backends_config_map[self.cp_backend]}, but cp_handler was set to {type(self.cp_handler)}" + ) + + if self.sp_size > 1: + if self.sp_handler is None: + self.sp_handler = DeepSpeedSequenceParallelConfig() + if self.dp_replicate_size < 1: + raise ValueError(f"dp_replicate_size must be at least 1, but got {self.dp_replicate_size}") + if self.dp_shard_size < 1: + raise ValueError(f"dp_shard_size must be at least 1, but got {self.dp_shard_size}") + if self.tp_size < 1: + raise ValueError(f"tp_size must be at least 1, but got {self.tp_size}") + if self.cp_size < 1: + raise ValueError(f"cp_size must be at least 1, but got {self.cp_size}") + valid_cp_backends = ["torch"] + if self.cp_backend not in valid_cp_backends: + raise ValueError(f"cp_backend must be one of {valid_cp_backends}, but got {self.cp_backend}") + + if self.sp_size < 1: + raise ValueError(f"sp_size must be at least 1, but got {self.sp_size}") + valid_sp_backends = ["deepspeed"] + if self.sp_backend not in valid_sp_backends: + raise ValueError(f"sp_backend must be one of {valid_sp_backends}, but got {self.sp_backend}") + + # CP and SP are mutually exclusive + if self.cp_size > 1 and self.sp_size > 1: + raise ValueError( + "Context Parallelism (CP) and Sequence Parallelism (SP) are mutually exclusive. " + f"Got cp_size={self.cp_size} and sp_size={self.sp_size}. " + "Please set either cp_size=1 or sp_size=1." + ) + + if (self.tp_size > 1 or self.cp_size > 1) and self.dp_replicate_size > 1 and self.dp_shard_size == 1: + raise ValueError( + "Tensor/Context parallelism (tp/cp_size > 1) cannot be used with pure data parallelism (dp_replicate_size > 1 and dp_shard_size == 1). " + "Please set dp_shard_size > 1 and dp_replicate_size == 1 to compose FSDP + TP/CP for 2D parallel, " + "or set dp_replicate_size == 1 and dp_shard_size > 1 to compose HSDP + TP/CP for 3D parallel." + ) + self._sizes = { + "dp_replicate": self.dp_replicate_size, + "dp_shard": self.dp_shard_size, + "tp": self.tp_size, + "cp": self.cp_size, + "sp": self.sp_size, + } + + def _set_size(self, parallelism: str, size: int): + assert parallelism in self._sizes.keys(), f"Parallelism must be one of {self._sizes.keys()}" + self._sizes[parallelism] = size + setattr(self, f"{parallelism}_size", size) + + def _validate_accelerator(self, accelerator: "Accelerator"): + _warnings = set() + if not accelerator.multi_device and self.total_size == 1: + # No distributed setup, valid parallelism config + return + + # We need this to ensure DDP works + if self.total_size == 1: + self._set_size("dp_replicate", accelerator.num_processes) + + # For DeepSpeed SP, DeepSpeed handles global process groups internally. + # Skip the total_size == num_processes validation since: + # 1. DeepSpeed manages SP groups globally via initialize_sequence_parallel() + # 2. num_processes is per-node in multi-node, but total_size is local parallelism config + # 3. The actual global parallelism (SP × DP) is handled by DeepSpeed's process groups + if self.sp_backend == "deepspeed" and self.sp_size > 1: + pass + elif self.total_size != accelerator.num_processes: + raise ValueError( + f"ParallelismConfig total_size ({self.total_size}) does not match " + f"num_processes ({accelerator.num_processes}). Please adjust dp_replicate_size/ " + f"dp_shard_size/tp_size/cp_size/sp_size." + ) + + if self.total_size > 1 and not ( + accelerator.is_fsdp2 + or accelerator.multi_device + or accelerator.distributed_type == DistributedType.DEEPSPEED + ): + raise ValueError( + f"ParallelismConfig is only compatible DistributedType.FSDP (version 2) or DistributedType.Multi{{Device}} or DistributedType.DEEPSPEED, but got {accelerator.distributed_type}." + ) + + for parallelism, size in self._sizes.items(): + if size == 1 and getattr(self, f"{parallelism}_handler", None) is not None: + _warnings.add( + f"ParallelismConfig.{parallelism}_handler is set, but {parallelism}_size is set to 1. This handler will be ignored." + ) + + if _warnings and accelerator.is_main_process: + warnings.warn( + "ParallelismConfig has the following warnings:\n" + "\n".join(_warnings), + UserWarning, + ) diff --git a/kbidu_env/Lib/site-packages/accelerate/scheduler.py b/kbidu_env/Lib/site-packages/accelerate/scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..1fa8a13f238afd7b908ee8e8cb8e0620f48d4ff8 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/scheduler.py @@ -0,0 +1,98 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation + +import warnings + +from .state import AcceleratorState, GradientState + + +warnings.filterwarnings("ignore", category=UserWarning, module="torch.optim.lr_scheduler") + + +class AcceleratedScheduler: + """ + A wrapper around a learning rate scheduler that will only step when the optimizer(s) have a training step. Useful + to avoid making a scheduler step too fast when gradients went overflow and there was no training step (in mixed + precision training) + + When performing gradient accumulation scheduler lengths should not be changed accordingly, Accelerate will always + step the scheduler to account for it. + + Args: + scheduler (`torch.optim.lr_scheduler._LRScheduler`): + The scheduler to wrap. + optimizers (one or a list of `torch.optim.Optimizer`): + The optimizers used. + step_with_optimizer (`bool`, *optional*, defaults to `True`): + Whether or not the scheduler should be stepped at each optimizer step. + split_batches (`bool`, *optional*, defaults to `False`): + Whether or not the dataloaders split one batch across the different processes (so batch size is the same + regardless of the number of processes) or create batches on each process (so batch size is the original + batch size multiplied by the number of processes). + """ + + def __init__(self, scheduler, optimizers, step_with_optimizer: bool = True, split_batches: bool = False): + self.scheduler = scheduler + self.optimizers = optimizers if isinstance(optimizers, (list, tuple)) else [optimizers] + self.split_batches = split_batches + self.step_with_optimizer = step_with_optimizer + self.gradient_state = GradientState() + + def step(self, *args, **kwargs): + if not self.step_with_optimizer: + # No link between scheduler and optimizer -> just step + self.scheduler.step(*args, **kwargs) + return + + # Otherwise, first make sure the optimizer was stepped. + if not self.gradient_state.sync_gradients: + if self.gradient_state.adjust_scheduler: + self.scheduler._step_count += 1 + return + + for opt in self.optimizers: + if opt.step_was_skipped: + return + if self.split_batches: + # Split batches -> the training dataloader batch size is not changed so one step per training step + self.scheduler.step(*args, **kwargs) + else: + # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do + # num_processes steps per training step + num_processes = AcceleratorState().num_processes + for _ in range(num_processes): + # Special case when using OneCycle and `drop_last` was not used + if hasattr(self.scheduler, "total_steps"): + if self.scheduler._step_count <= self.scheduler.total_steps: + self.scheduler.step(*args, **kwargs) + else: + self.scheduler.step(*args, **kwargs) + + # Passthroughs + def get_last_lr(self): + return self.scheduler.get_last_lr() + + def state_dict(self): + return self.scheduler.state_dict() + + def load_state_dict(self, state_dict): + self.scheduler.load_state_dict(state_dict) + + def get_lr(self): + return self.scheduler.get_lr() + + def print_lr(self, *args, **kwargs): + return self.scheduler.print_lr(*args, **kwargs) diff --git a/kbidu_env/Lib/site-packages/accelerate/state.py b/kbidu_env/Lib/site-packages/accelerate/state.py new file mode 100644 index 0000000000000000000000000000000000000000..aeb8c8b671f818762b545d5288c9efef716b490e --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/state.py @@ -0,0 +1,1371 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import logging +import os +import threading +import warnings +import weakref +from contextlib import contextmanager +from functools import partial +from typing import Any, Callable + +import torch + +from .utils import ( + DistributedType, + DynamoBackend, + GradientAccumulationPlugin, + check_cuda_fp8_capability, + check_cuda_p2p_ib_support, + deepspeed_required, + get_cpu_distributed_information, + get_int_from_env, + is_datasets_available, + is_deepspeed_available, + is_fp8_available, + is_habana_gaudi1, + is_hpu_available, + is_mlu_available, + is_mps_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_torch_xla_available, + is_xccl_available, + is_xpu_available, + parse_choice_from_env, + parse_flag_from_env, + set_numa_affinity, +) +from .utils.dataclasses import SageMakerDistributedType + + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + import torch_xla.runtime as xr + +if is_mlu_available(check_device=False): + import torch_mlu # noqa: F401 + +if is_sdaa_available(check_device=False): + import torch_sdaa # noqa: F401 + +if is_musa_available(check_device=False): + import torch_musa # noqa: F401 + +if is_npu_available(check_device=False): + import torch_npu # noqa: F401 + + +logger = logging.getLogger(__name__) + + +def is_initialized() -> bool: + """ + Checks if the `AcceleratorState` has been initialized from `Accelerator`. Same as `AcceleratorState.initialized`, + but works as a module method. + """ + return AcceleratorState._shared_state != {} + + +# Lambda function that does nothing +def do_nothing(*args, **kwargs): + return None + + +class ThreadLocalSharedDict(threading.local): + """ + Descriptor that holds a dict shared between instances of a class in the same thread. + + Note: Descriptors have slightly different semantics than just a dict field on its own. + `PartialState(...)._shared_state` and `PartialState._shared_state` (instance vs class) give the same value: the + underlying _storage dict. Likewise, `PartialState(...)._shared_state = {...}` overrides the _storage dict inside + the descriptor as you would expect. However, `PartialState._shared_state = {}` actually replaces the descriptor + object with a dict instead Thus, you should modify the _storage dict in-place (e.g. `_shared_state.clear()`). + + See Python documentation for an explanation of descriptors: https://docs.python.org/3/howto/descriptor.html + + This is required for using PyTorch/XLA with PJRT in multithreaded mode (required for TPU v2 and v3). + + See https://github.com/pytorch/xla/blob/r2.0/docs/pjrt.md#multithreading-on-tpu-v2v3 + """ + + def __init__(self, thread_local: bool = False): + self._storage = {} + + def __get__(self, obj, objtype=None): + return self._storage + + def __set__(self, obj, value): + self._storage = value + + +# Prefer global shared dictionary, except when using TPU. +SharedDict = dict if not is_torch_xla_available() else ThreadLocalSharedDict + + +# Inspired by Alex Martelli's 'Borg'. +class PartialState: + """ + Singleton class that has information about the current training environment and functions to help with process + control. Designed to be used when only process control and device execution states are needed. Does *not* need to + be initialized from `Accelerator`. + + Args: + cpu (`bool`, *optional*): + Whether or not to force the script to execute on CPU. Will ignore any accelerators available if set to + `True` and force the execution on the CPU. + kwargs (additional keyword arguments, *optional*): + Additional keyword arguments to pass to the relevant `init_process_group` function. Valid `kwargs` can be + found in [`utils.InitProcessGroupKwargs`]. See the example section for detailed usage. + + **Available attributes:** + + - **device** (`torch.device`) -- The device to use. + - **distributed_type** ([`~accelerate.state.DistributedType`]) -- The type of distributed environment currently + in use. + - **local_process_index** (`int`) -- The index of the current process on the current server. + - **mixed_precision** (`str`) -- Whether or not the current script will use mixed precision, and if so the type + of mixed precision being performed. (Choose from 'no','fp16','bf16 or 'fp8'). + - **num_processes** (`int`) -- The number of processes currently launched in parallel. + - **process_index** (`int`) -- The index of the current process. + - **is_last_process** (`bool`) -- Whether or not the current process is the last one. + - **is_main_process** (`bool`) -- Whether or not the current process is the main one. + - **is_local_main_process** (`bool`) -- Whether or not the current process is the main one on the local node. + - **debug** (`bool`) -- Whether or not the current script is being run in debug mode. + + Example: + ```python + from accelerate.utils import InitProcessGroupKwargs + + # To include `InitProcessGroupKwargs`, init then call `.to_kwargs()` + kwargs = InitProcessGroupKwargs(...).to_kwargs() + state = PartialState(**kwargs) + ``` + """ + + _shared_state = SharedDict() + _known_attrs = [ + "_cpu", + "_mixed_precision", + "_shared_state", + "backend", + "debug", + "device", + "distributed_type", + "fork_launched", + "local_process_index", + "num_processes", + "process_index", + ] + + def __init__(self, cpu: bool = False, **kwargs): + self.__dict__ = self._shared_state + if not self.initialized: + self._cpu = cpu + self.backend = None + env_device = os.environ.get("ACCELERATE_TORCH_DEVICE", None) + self.device = torch.device(env_device) if env_device is not None else None + self.debug = parse_flag_from_env("ACCELERATE_DEBUG_MODE") + use_sagemaker_dp = kwargs.pop("_use_sagemaker_dp", None) + dist_information = None + if use_sagemaker_dp is None: + use_sagemaker_dp = ( + os.environ.get("ACCELERATE_USE_SAGEMAKER", "false").lower() == "true" + and os.environ.get("ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPE") != SageMakerDistributedType.NO + ) + + # Sets up self.backend + imports + original_backend = kwargs.pop("backend", None) + backend, distributed_type = self._prepare_backend(cpu, use_sagemaker_dp, original_backend) + if original_backend is not None and backend != original_backend: + raise ValueError(f"Your assigned backend {original_backend} is not available, please use {backend}") + self.backend = backend + self.distributed_type = distributed_type + use_deepspeed = False + if not cpu and self.backend != "xla": + if int(os.environ.get("LOCAL_RANK", -1)) != -1: + # Deal with spawning deepspeed + if os.environ.get("ACCELERATE_USE_DEEPSPEED", "false").lower() == "true": + if not is_deepspeed_available(): + raise ImportError( + "DeepSpeed is not available => install it using `pip3 install deepspeed` or build it from source" + ) + from deepspeed import comm as dist + + if not dist.is_initialized(): + if self.backend == "tccl": + local_rank = os.environ.get("LOCAL_RANK", -1) + torch.sdaa.set_device(f"sdaa:{local_rank}") + dist.init_distributed(dist_backend=self.backend, auto_mpi_discovery=False, **kwargs) + # We need to flag to `use_deepspeed` to be True to override `distributed_type` later + use_deepspeed = True + # Deal with all other backends but CPU, that gets handled special later + elif ( + self.distributed_type is not DistributedType.MULTI_CPU + and not torch.distributed.is_initialized() + ): + if self.backend == "tccl": + local_rank = os.environ.get("LOCAL_RANK", -1) + torch.sdaa.set_device(f"sdaa:{local_rank}") + if ( + self.backend == "nccl" + and os.environ.get("ACCELERATE_USE_FSDP", "false").lower() == "true" + and ( + os.environ.get("FSDP_OFFLOAD_PARAMS", "false").lower() == "true" + or os.environ.get("FSDP_STATE_DICT_TYPE", "SHARDED_STATE_DICT") == "FULL_STATE_DICT" + ) + ): + self.backend = "cuda:nccl,cpu:gloo" + if ( + self.backend == "xccl" + and os.environ.get("ACCELERATE_USE_FSDP", "false").lower() == "true" + and ( + os.environ.get("FSDP_OFFLOAD_PARAMS", "false").lower() == "true" + or os.environ.get("FSDP_STATE_DICT_TYPE", "SHARDED_STATE_DICT") == "FULL_STATE_DICT" + ) + ): + self.backend = "xpu:xccl,cpu:gloo" + torch.distributed.init_process_group(backend=self.backend, **kwargs) + + # CPU require special env configs to be set + if self.distributed_type == DistributedType.MULTI_CPU: + dist_information = get_cpu_distributed_information() + os.environ["RANK"] = str(dist_information.rank) + os.environ["WORLD_SIZE"] = str(dist_information.world_size) + os.environ["LOCAL_RANK"] = str(dist_information.local_rank) + os.environ["LOCAL_WORLD_SIZE"] = str(dist_information.local_world_size) + if not os.environ.get("MASTER_PORT", None): + os.environ["MASTER_PORT"] = "29500" + if ( + not os.environ.get("MASTER_ADDR", None) + and dist_information.local_world_size != dist_information.world_size + and self.backend != "mpi" + ): + raise ValueError( + "Tried to launch on distributed with multinode, but `MASTER_ADDR` env was not set, " + "please try exporting rank 0's hostname as `MASTER_ADDR`" + ) + kwargs["rank"] = dist_information.rank + kwargs["world_size"] = dist_information.world_size + + if ( + self.distributed_type == DistributedType.MULTI_CPU + and get_int_from_env(["OMP_NUM_THREADS"], 0) == 0 + ): + import psutil + + num_cpu_threads_per_process = int( + psutil.cpu_count(logical=False) / dist_information.local_world_size + ) + if num_cpu_threads_per_process == 0: + num_cpu_threads_per_process = 1 + torch.set_num_threads(num_cpu_threads_per_process) + warnings.warn( + f"OMP_NUM_THREADS/MKL_NUM_THREADS unset, we set it at {num_cpu_threads_per_process} to improve oob" + " performance." + ) + + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group(backend=self.backend, **kwargs) + + # No backend == no distributed training + if self.backend is None: + self.distributed_type = DistributedType.NO + self.num_processes = 1 + self.process_index = 0 + self.local_process_index = 0 + elif self.backend == "xla": + # XLA needs device setting first for `set_replication` + self.set_device() + xm.set_replication(self.device, xm.get_xla_supported_devices()) + self.num_processes = xr.world_size() + self.process_index = xr.global_ordinal() + if is_torch_xla_available(check_is_tpu=True): + self.local_process_index = xm.get_local_ordinal() + else: + self.local_process_index = int(os.environ.get("LOCAL_RANK", -1)) + else: + self.num_processes = torch.distributed.get_world_size() + self.process_index = torch.distributed.get_rank() + self.local_process_index = ( + int(os.environ.get("LOCAL_RANK", -1)) if dist_information is None else dist_information.local_rank + ) + self.set_device() + # Now we can change to deepseed + if use_deepspeed: + self.distributed_type = DistributedType.DEEPSPEED + + # Set CPU affinity if enabled + if parse_flag_from_env("ACCELERATE_CPU_AFFINITY", False): + set_numa_affinity(self.local_process_index) + + # Check for old RTX 4000's that can't use P2P or IB and are on old drivers + if self.device.type == "cuda" and not check_cuda_p2p_ib_support(): + if "NCCL_P2P_DISABLE" not in os.environ or "NCCL_IB_DISABLE" not in os.environ: + raise NotImplementedError( + "Using RTX 4000 series doesn't support faster communication broadband via P2P or IB. " + 'Please set `NCCL_P2P_DISABLE="1"` and `NCCL_IB_DISABLE="1" or use `accelerate launch` which ' + "will do this automatically." + ) + + # Important: This should be the *only* code outside of `self.initialized!` + self.fork_launched = parse_flag_from_env("FORK_LAUNCHED", 0) + + def __repr__(self) -> str: + return ( + f"Distributed environment: {self.distributed_type}{(' Backend: ' + self.backend) if self.backend else ''}\n" + f"Num processes: {self.num_processes}\n" + f"Process index: {self.process_index}\n" + f"Local process index: {self.local_process_index}\n" + f"Device: {self.device}\n" + ) + + @staticmethod + def _reset_state(): + "Resets `_shared_state`, is used internally and should not be called" + PartialState._shared_state.clear() + + @property + def initialized(self) -> bool: + "Returns whether the `PartialState` has been initialized" + return self._shared_state != {} + + @property + def use_distributed(self): + """ + Whether the Accelerator is configured for distributed training + """ + return self.distributed_type != DistributedType.NO and self.num_processes > 1 + + @property + def is_last_process(self) -> bool: + "Returns whether the current process is the last one" + return self.process_index == self.num_processes - 1 + + @property + def is_main_process(self) -> bool: + "Returns whether the current process is the main process" + return ( + self.process_index == 0 if self.distributed_type != DistributedType.MEGATRON_LM else self.is_last_process + ) + + @property + def is_local_main_process(self) -> bool: + "Returns whether the current process is the main process on the local node" + return ( + self.local_process_index == 0 + if self.distributed_type != DistributedType.MEGATRON_LM + else self.is_last_process + ) + + def wait_for_everyone(self): + """ + Will stop the execution of the current process until every other process has reached that point (so this does + nothing when the script is only run in one process). Useful to do before saving a model. + + Example: + + ```python + >>> # Assuming two GPU processes + >>> import time + >>> from accelerate.state import PartialState + + >>> state = PartialState() + >>> if state.is_main_process: + ... time.sleep(2) + >>> else: + ... print("I'm waiting for the main process to finish its sleep...") + >>> state.wait_for_everyone() + >>> # Should print on every process at the same time + >>> print("Everyone is here") + ``` + """ + if self.distributed_type in ( + DistributedType.MULTI_GPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_CPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + DistributedType.DEEPSPEED, + DistributedType.FSDP, + ): + torch.distributed.barrier(device_ids=[self.local_process_index]) + elif self.distributed_type == DistributedType.XLA: + xm.rendezvous("accelerate.utils.wait_for_everyone") + + def _goes_first(self, is_main: bool): + if not is_main: + self.wait_for_everyone() + + yield + + if is_main: + self.wait_for_everyone() + + @contextmanager + def split_between_processes(self, inputs: list | tuple | dict | torch.Tensor, apply_padding: bool = False): + """ + Splits `input` between `self.num_processes` quickly and can be then used on that process. Useful when doing + distributed inference, such as with different prompts. + + Note that when using a `dict`, all keys need to have the same number of elements. + + Args: + inputs (`list`, `tuple`, `torch.Tensor`, `dict` of `list`/`tuple`/`torch.Tensor`, or `datasets.Dataset`): + The input to split between processes. + apply_padding (`bool`, `optional`, defaults to `False`): + Whether to apply padding by repeating the last element of the input so that all processes have the same + number of elements. Useful when trying to perform actions such as `gather()` on the outputs or passing + in less inputs than there are processes. If so, just remember to drop the padded elements afterwards. + + + Example: + + ```python + # Assume there are two processes + from accelerate import PartialState + + state = PartialState() + with state.split_between_processes(["A", "B", "C"]) as inputs: + print(inputs) + # Process 0 + ["A", "B"] + # Process 1 + ["C"] + + with state.split_between_processes(["A", "B", "C"], apply_padding=True) as inputs: + print(inputs) + # Process 0 + ["A", "B"] + # Process 1 + ["C", "C"] + ``` + """ + if self.num_processes == 1: + yield inputs + return + length = len(inputs) + # Nested dictionary of any types + if isinstance(inputs, dict): + length = len(inputs[list(inputs.keys())[0]]) + if not all(len(v) == length for v in inputs.values()): + raise ValueError("All values in the dictionary must have the same length") + num_samples_per_process, num_extras = divmod(length, self.num_processes) + start_index = self.process_index * num_samples_per_process + min(self.process_index, num_extras) + end_index = start_index + num_samples_per_process + (1 if self.process_index < num_extras else 0) + + def _split_values(inputs, start_index, end_index): + if isinstance(inputs, (list, tuple, torch.Tensor)): + result = inputs[start_index:end_index] + if apply_padding: + if isinstance(result, torch.Tensor): + from accelerate.utils import pad_across_processes, send_to_device + + # The tensor needs to be on the device before we can pad it + tensorized_result = send_to_device(result, self.device) + result = pad_across_processes( + tensorized_result, pad_index=send_to_device(inputs[-1], self.device) + ) + else: + result += [inputs[-1]] * (num_samples_per_process + (1 if num_extras > 0 else 0) - len(result)) + return result + elif isinstance(inputs, dict): + for key in inputs.keys(): + inputs[key] = _split_values(inputs[key], start_index, end_index) + return inputs + else: + if is_datasets_available(): + from datasets import Dataset + + if isinstance(inputs, Dataset): + clamped_start = min(start_index, len(inputs)) + clamped_end = min(end_index, len(inputs)) + result_idcs = list(range(clamped_start, clamped_end)) + if apply_padding: + result_idcs += [len(inputs) - 1] * ( + num_samples_per_process + (1 if num_extras > 0 else 0) - len(result_idcs) + ) + return inputs.select(result_idcs) + return inputs + + yield _split_values(inputs, start_index, end_index) + + @contextmanager + def main_process_first(self): + """ + Lets the main process go first inside a with block. + + The other processes will enter the with block after the main process exits. + + Example: + + ```python + >>> from accelerate import Accelerator + + >>> accelerator = Accelerator() + >>> with accelerator.main_process_first(): + ... # This will be printed first by process 0 then in a seemingly + ... # random order by the other processes. + ... print(f"This will be printed by process {accelerator.process_index}") + ``` + """ + yield from self._goes_first(self.is_main_process) + + @contextmanager + def local_main_process_first(self): + """ + Lets the local main process go inside a with block. + + The other processes will enter the with block after the main process exits. + + Example: + + ```python + >>> from accelerate.state import PartialState + + >>> state = PartialState() + >>> with state.local_main_process_first(): + ... # This will be printed first by local process 0 then in a seemingly + ... # random order by the other processes. + ... print(f"This will be printed by process {state.local_process_index}") + ``` + """ + yield from self._goes_first(self.is_local_main_process) + + def on_main_process(self, function: Callable[..., Any] | None = None): + """ + Decorator that only runs the decorated function on the main process. + + Args: + function (`Callable`): The function to decorate. + + Example: + + ```python + >>> from accelerate.state import PartialState + + >>> state = PartialState() + + + >>> @state.on_main_process + ... def print_something(): + ... print("This will be printed by process 0 only.") + + + >>> print_something() + "This will be printed by process 0 only" + ``` + """ + if not self.initialized: + raise ValueError("The `PartialState` or `Accelerator` must be initialized before calling this function.") + if self.is_main_process or not self.use_distributed: + return function + return do_nothing + + def on_local_main_process(self, function: Callable[..., Any] | None = None): + """ + Decorator that only runs the decorated function on the local main process. + + Args: + function (`Callable`): The function to decorate. + + Example: + ```python + # Assume we have 2 servers with 4 processes each. + from accelerate.state import PartialState + + state = PartialState() + + + @state.on_local_main_process + def print_something(): + print("This will be printed by process 0 only on each server.") + + + print_something() + # On server 1: + "This will be printed by process 0 only" + # On server 2: + "This will be printed by process 0 only" + ``` + """ + if self.is_local_main_process or not self.use_distributed: + return function + return do_nothing + + def on_last_process(self, function: Callable[..., Any]): + """ + Decorator that only runs the decorated function on the last process. + + Args: + function (`Callable`): The function to decorate. + + Example: + ```python + # Assume we have 4 processes. + from accelerate.state import PartialState + + state = PartialState() + + + @state.on_last_process + def print_something(): + print(f"Printed on process {state.process_index}") + + + print_something() + "Printed on process 3" + ``` + """ + if self.is_last_process or not self.use_distributed: + return function + return do_nothing + + def on_process(self, function: Callable[..., Any] | None = None, process_index: int | None = None): + """ + Decorator that only runs the decorated function on the process with the given index. + + Args: + function (`Callable`, `optional`): + The function to decorate. + process_index (`int`, `optional`): + The index of the process on which to run the function. + + Example: + ```python + # Assume we have 4 processes. + from accelerate.state import PartialState + + state = PartialState() + + + @state.on_process(process_index=2) + def print_something(): + print(f"Printed on process {state.process_index}") + + + print_something() + "Printed on process 2" + ``` + """ + if function is None: + return partial(self.on_process, process_index=process_index) + if (self.process_index == process_index) or (not self.use_distributed): + return function + return do_nothing + + def on_local_process(self, function: Callable[..., Any] | None = None, local_process_index: int | None = None): + """ + Decorator that only runs the decorated function on the process with the given index on the current node. + + Args: + function (`Callable`, *optional*): + The function to decorate. + local_process_index (`int`, *optional*): + The index of the local process on which to run the function. + + Example: + ```python + # Assume we have 2 servers with 4 processes each. + from accelerate import Accelerator + + accelerator = Accelerator() + + + @accelerator.on_local_process(local_process_index=2) + def print_something(): + print(f"Printed on process {accelerator.local_process_index}") + + + print_something() + # On server 1: + "Printed on process 2" + # On server 2: + "Printed on process 2" + ``` + """ + if function is None: + return partial(self.on_local_process, local_process_index=local_process_index) + if (self.local_process_index == local_process_index) or (not self.use_distributed): + return function + return do_nothing + + def print(self, *args, **kwargs): + if self.is_local_main_process: + print(*args, **kwargs) + + @property + def default_device(self) -> torch.device: + """ + Returns the default device which is: + - MPS if `torch.backends.mps.is_available()` and `torch.backends.mps.is_built()` both return True. + - CUDA if `torch.cuda.is_available()` + - MLU if `is_mlu_available()` + - SDAA if `is_sdaa_available()` + - MUSA if `is_musa_available()` + - NPU if `is_npu_available()` + - HPU if `is_hpu_available()` + - NEURON if `is_neuron_available()` + - CPU otherwise + """ + if is_mps_available(): + os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" + return torch.device("mps") + elif is_mlu_available(): + return torch.device("mlu") + elif is_sdaa_available(): + return torch.device("sdaa") + elif is_musa_available(): + return torch.device("musa") + # NPU should be checked before CUDA when using `transfer_to_npu` + # See issue #3020: https://github.com/huggingface/accelerate/issues/3020 + elif is_npu_available(): + return torch.device("npu") + elif is_hpu_available(): + return torch.device("hpu") + elif torch.cuda.is_available(): + return torch.device("cuda") + elif is_xpu_available(): + return torch.device("xpu") + elif is_neuron_available(): + return torch.device("neuron") + else: + return torch.device("cpu") + + def _prepare_backend( + self, cpu: bool = False, sagemaker_dp=False, backend: str | None = None + ) -> tuple[str, DistributedType]: + "Prepares any imports needed before initializing the distributed backend and sets `self.backend` properly" + distributed_type = None + if sagemaker_dp: + import smdistributed.dataparallel.torch.torch_smddp # noqa + + backend = "smddp" + distributed_type = DistributedType.MULTI_GPU + elif is_torch_xla_available(): + backend = "xla" + distributed_type = DistributedType.XLA + + elif int(os.environ.get("LOCAL_RANK", -1)) != -1 and not cpu: + if is_mlu_available(): + backend = "cncl" + distributed_type = DistributedType.MULTI_MLU + elif is_sdaa_available(): + backend = "tccl" + distributed_type = DistributedType.MULTI_SDAA + elif is_musa_available(): + backend = "mccl" + distributed_type = DistributedType.MULTI_MUSA + # NPU should be checked before CUDA when using `transfer_to_npu` + # See issue #3020: https://github.com/huggingface/accelerate/issues/3020 + elif is_npu_available(): + backend = "hccl" + distributed_type = DistributedType.MULTI_NPU + elif is_hpu_available(init_hccl=True): + if backend is None: + backend = "hccl" + distributed_type = DistributedType.MULTI_HPU + elif torch.cuda.is_available(): + if backend is None: + backend = "nccl" + distributed_type = DistributedType.MULTI_GPU + elif is_xpu_available() and is_xccl_available(): + if backend is None: + backend = "xccl" + distributed_type = DistributedType.MULTI_XPU + elif is_neuron_available(): + backend = "neuron" + distributed_type = DistributedType.MULTI_NEURON + + if ( + distributed_type is None + and cpu + and ( + int(os.environ.get("LOCAL_RANK", -1)) != -1 + or get_int_from_env(["PMI_SIZE", "OMPI_COMM_WORLD_SIZE", "MV2_COMM_WORLD_SIZE", "WORLD_SIZE"], 1) > 1 + ) + ): + distributed_type = DistributedType.MULTI_CPU + + if backend in (None, "mpi") and torch.distributed.is_mpi_available(): + backend = "mpi" + else: + backend = "gloo" + if distributed_type is None: + distributed_type = DistributedType.NO + + return backend, distributed_type + + def set_device(self): + """ + Sets the device in `self.device` to the current distributed environment. + """ + if self.device is not None: + return + if self.distributed_type == DistributedType.NO: + self.device = torch.device("cpu") if self._cpu else self.default_device + return + device = str(self.distributed_type).split(".")[-1].replace("MULTI_", "").lower() + if device not in ("cpu", "gpu", "mlu", "musa", "npu", "xpu", "xla", "hpu", "sdaa", "neuron"): + raise ValueError( + f"Can't set device for {self.distributed_type} ({device}), verify we should be calling `_set_device()` for it!" + ) + if device == "xla": + self.device = xm.xla_device() + elif device == "hpu": + self.device = torch.device("hpu", torch.hpu.current_device()) + else: + if device == "gpu": + device = "cuda" + device_module = getattr(torch, device) + device_index = self.local_process_index % device_module.device_count() + self.device = torch.device(device, device_index) + device_module.set_device(self.device) + + def destroy_process_group(self, group=None): + """ + Destroys the process group. If one is not specified, the default process group is destroyed. + """ + if self.fork_launched and group is None: + return + # needed when using torch.distributed.init_process_group + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group(group) + + def __getattr__(self, name: str): + # By this point we know that no attributes of `self` contain `name`, + # so we just modify the error message + if name in self._known_attrs: + raise AttributeError( + f"`PartialState` object has no attribute `{name}`. " + "This happens if `PartialState._reset_state()` was called and " + "an `Accelerator` or `PartialState` was not reinitialized." + ) + # Raise a typical AttributeError + raise AttributeError(f"'PartialState' object has no attribute '{name}'") + + +class AcceleratorState: + """ + Singleton class that has information about the current training environment. + + **Available attributes:** + + - **device** (`torch.device`) -- The device to use. + - **distributed_type** ([`~accelerate.state.DistributedType`]) -- The type of distributed environment currently + in use. + - **parallelism_config** ([`~accelerate.utils.ParallelismConfig`]) -- The parallelism configuration for the + current training environment. This is used to configure the distributed training environment. + - **initialized** (`bool`) -- Whether or not the `AcceleratorState` has been initialized from `Accelerator`. + - **local_process_index** (`int`) -- The index of the current process on the current server. + - **mixed_precision** (`str`) -- Whether or not the current script will use mixed precision, and if so the type + of mixed precision being performed. (Choose from 'no','fp16','bf16 or 'fp8'). + - **num_processes** (`int`) -- The number of processes currently launched in parallel. + - **process_index** (`int`) -- The index of the current process. + - **is_last_process** (`bool`) -- Whether or not the current process is the last one. + - **is_main_process** (`bool`) -- Whether or not the current process is the main one. + - **is_local_main_process** (`bool`) -- Whether or not the current process is the main one on the local node. + - **debug** (`bool`) -- Whether or not the current script is being run in debug mode. + """ + + _shared_state = SharedDict() + _known_attrs = PartialState._known_attrs + [ + "deepspeed_plugin", + "fsdp_plugin", + "megatron_lm_plugin", + "dynamo_plugin", + ] + + def __init__( + self, + mixed_precision: str | None = None, + cpu: bool = False, + dynamo_plugin=None, + deepspeed_plugin=None, + fsdp_plugin=None, + torch_tp_plugin=None, + megatron_lm_plugin=None, + parallelism_config=None, + _from_accelerator: bool = False, + **kwargs, + ): + self.__dict__ = self._shared_state + if parse_flag_from_env("ACCELERATE_USE_CPU"): + cpu = True + if PartialState._shared_state == {}: + PartialState(cpu, **kwargs) + self.__dict__.update(PartialState._shared_state) + self._check_initialized(mixed_precision, cpu) + if not self.initialized: + self.deepspeed_plugins = None + self.torch_tp_plugin = torch_tp_plugin + self.parallelism_config = parallelism_config + self.device_mesh = None + mixed_precision = ( + parse_choice_from_env("ACCELERATE_MIXED_PRECISION", "no") + if mixed_precision is None + else mixed_precision.lower() + ) + if mixed_precision == "fp8": + # this is confusing, why is is_fp8_available only checks for library availability ? + if not is_fp8_available(): + raise ValueError( + "Using `fp8` precision requires `transformer_engine` or `MS-AMP` to be installed." + ) + elif torch.cuda.is_available() and not check_cuda_fp8_capability(): + logger.warning( + f"The current device has compute capability of {torch.cuda.get_device_capability()} which is " + "insufficient for FP8 mixed precision training (requires a GPU Hopper/Ada Lovelace " + "or higher, compute capability of 8.9 or higher). Will use FP16 instead." + ) + mixed_precision = "fp16" + elif is_habana_gaudi1(): + logger.warning( + "The current HPU device is Gaudi1 which does not support FP8 mixed precision training (requires " + "Gaudi2 or higher). Will use BF16 instead." + ) + mixed_precision = "bf16" + + self.dynamo_plugin = dynamo_plugin + if not _from_accelerator: + raise ValueError( + "Please make sure to properly initialize your accelerator via `accelerator = Accelerator()` " + "before using any functionality from the `accelerate` library." + ) + # deepspeed handles mixed_precision using deepspeed_config. But we need to set it to fp8 + # if we're using fp8. + if self.distributed_type == DistributedType.DEEPSPEED and mixed_precision != "fp8": + self._mixed_precision = "no" + else: + self._mixed_precision = mixed_precision + + if self.distributed_type == DistributedType.XLA and is_torch_xla_available(check_is_tpu=True): + if mixed_precision == "bf16": + if os.environ.get("ACCELERATE_DOWNCAST_BF16"): + os.environ["XLA_USE_BF16"] = str(0) + os.environ["XLA_DOWNCAST_BF16"] = str(1) + self.downcast_bfloat = True + else: + os.environ["XLA_USE_BF16"] = str(1) + os.environ["XLA_DOWNCAST_BF16"] = str(0) + self.downcast_bfloat = False + elif os.environ.get("ACCELERATE_USE_DEEPSPEED", "false").lower() == "true" and not cpu: + self.distributed_type = DistributedType.DEEPSPEED + if not isinstance(deepspeed_plugin, dict): + deepspeed_plugin.set_mixed_precision(mixed_precision) + deepspeed_plugin.select(_from_accelerator_state=True) + else: + for plugin in deepspeed_plugin.values(): + plugin.set_mixed_precision(mixed_precision) + # The first plugin passed in is always the active one + first_plugin = next(iter(deepspeed_plugin.values())) + first_plugin.select(_from_accelerator_state=True) + self.deepspeed_plugins = deepspeed_plugin + elif self.distributed_type in [ + DistributedType.MULTI_GPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + ]: + # TODO: Siro - remove when axolotl fixes their side + if not os.environ.get("ACCELERATE_ALLOW_CP_STANDALONE", "false").lower() == "true": + if self.parallelism_config and self.parallelism_config.cp_enabled and fsdp_plugin is None: + raise ValueError( + "`cp_size > 1` specified in the `parallelism_config`, but no `fsdp_plugin` was provided. We need a `fsdp_plugin` to use context parallelism with `cp_backend=torch`, as we also shard the model across the device mesh to save more memory" + ) + if ( + self.parallelism_config is not None + and self.parallelism_config.cp_enabled + and fsdp_plugin.fsdp_version == 1 + ): + raise ValueError( + "Using `cp_size>1` requires FSDP2, but the provided `fsdp_plugin` is using FSDP1. " + ) + if (os.environ.get("ACCELERATE_USE_FSDP", "false").lower() == "true" or fsdp_plugin is not None) or ( + self.parallelism_config is not None and self.parallelism_config.cp_enabled + ): + self.distributed_type = DistributedType.FSDP + if self._mixed_precision != "no" and fsdp_plugin is not None: + fsdp_plugin.set_mixed_precision(self._mixed_precision) + self.fsdp_plugin = fsdp_plugin + if os.environ.get( + "ACCELERATE_USE_MEGATRON_LM", "false" + ).lower() == "true" and self.distributed_type not in [ + DistributedType.MULTI_XPU, + ]: + self.distributed_type = DistributedType.MEGATRON_LM + megatron_lm_plugin.set_mixed_precision(self._mixed_precision) + self.megatron_lm_plugin = megatron_lm_plugin + if ( + self.dynamo_plugin.backend != DynamoBackend.NO + and self._mixed_precision == "no" + and self.device.type == "cuda" + ): + torch.backends.cuda.matmul.allow_tf32 = True + if ( + self.dynamo_plugin.backend != DynamoBackend.NO + and self._mixed_precision == "no" + and self.device.type == "musa" + ): + torch.backends.musa.matmul.allow_tf32 = True + PartialState._shared_state["distributed_type"] = self.distributed_type + + @property + def initialized(self) -> bool: + return self._shared_state != PartialState._shared_state + + def __repr__(self): + repr = PartialState().__repr__() + f"\nMixed precision type: {self.mixed_precision}\n" + if self.distributed_type == DistributedType.DEEPSPEED: + repr += f"ds_config: {self.deepspeed_plugin.deepspeed_config}\n" + return repr + + def _check_initialized(self, mixed_precision=None, cpu=None): + "Checks if a modification is trying to be made and the `AcceleratorState` has already been initialized" + if self.initialized: + err = "AcceleratorState has already been initialized and cannot be changed, restart your runtime completely and pass `{flag}` to `Accelerator()`." + if cpu and self.device.type != "cpu": + raise ValueError(err.format(flag="cpu=True")) + if ( + mixed_precision is not None + and mixed_precision != self._mixed_precision + and self.distributed_type != DistributedType.DEEPSPEED + ): + raise ValueError(err.format(flag=f"mixed_precision='{mixed_precision}'")) + + @property + def mixed_precision(self): + if self.distributed_type == DistributedType.DEEPSPEED and self._mixed_precision != "fp8": + config = self.deepspeed_plugin.deepspeed_config + if config.get("fp16", {}).get("enabled", False): + mixed_precision = "fp16" + elif config.get("bf16", {}).get("enabled", False): + mixed_precision = "bf16" + else: + mixed_precision = "no" + else: + mixed_precision = self._mixed_precision + return mixed_precision + + @staticmethod + def _reset_state(reset_partial_state: bool = False): + "Resets `_shared_state`, is used internally and should not be called" + AcceleratorState._shared_state.clear() + if reset_partial_state: + PartialState._reset_state() + + def destroy_process_group(self, group=None): + """ + Destroys the process group. If one is not specified, the default process group is destroyed. + + If `self.fork_launched` is `True` and `group` is `None`, nothing happens. + """ + PartialState().destroy_process_group(group) + + @property + def fork_launched(self): + return PartialState().fork_launched + + @property + def use_distributed(self): + """ + Whether the Accelerator is configured for distributed training + """ + return PartialState().use_distributed + + @property + def is_fsdp2(self) -> bool: + return self.distributed_type == DistributedType.FSDP and self.fsdp_plugin.fsdp_version == 2 + + @property + def is_last_process(self) -> bool: + "Returns whether the current process is the last one" + return PartialState().is_last_process + + @property + def is_main_process(self) -> bool: + "Returns whether the current process is the main process" + return PartialState().is_main_process + + @property + def is_local_main_process(self) -> bool: + "Returns whether the current process is the main process on the local node" + return PartialState().is_local_main_process + + def wait_for_everyone(self): + PartialState().wait_for_everyone() + + @contextmanager + def split_between_processes(self, inputs: list | tuple | dict | torch.Tensor, apply_padding: bool = False): + """ + Splits `input` between `self.num_processes` quickly and can be then used on that process. Useful when doing + distributed inference, such as with different prompts. + + Note that when using a `dict`, all keys need to have the same number of elements. + + Args: + inputs (`list`, `tuple`, `torch.Tensor`, or `dict` of `list`/`tuple`/`torch.Tensor`): + The input to split between processes. + apply_padding (`bool`, `optional`, defaults to `False`): + Whether to apply padding by repeating the last element of the input so that all processes have the same + number of elements. Useful when trying to perform actions such as `gather()` on the outputs or passing + in less inputs than there are processes. If so, just remember to drop the padded elements afterwards. + + + Example: + + ```python + # Assume there are two processes + from accelerate.state import AcceleratorState + + state = AcceleratorState() + with state.split_between_processes(["A", "B", "C"]) as inputs: + print(inputs) + # Process 0 + ["A", "B"] + # Process 1 + ["C"] + + with state.split_between_processes(["A", "B", "C"], apply_padding=True) as inputs: + print(inputs) + # Process 0 + ["A", "B"] + # Process 1 + ["C", "C"] + ``` + """ + with PartialState().split_between_processes(inputs, apply_padding=apply_padding) as inputs: + yield inputs + + @contextmanager + def main_process_first(self): + """ + Lets the main process go first inside a with block. + + The other processes will enter the with block after the main process exits. + """ + with PartialState().main_process_first(): + yield + + @contextmanager + def local_main_process_first(self): + """ + Lets the local main process go inside a with block. + + The other processes will enter the with block after the main process exits. + """ + with PartialState().local_main_process_first(): + yield + + @property + def deepspeed_plugin(self): + """ + Returns the currently active DeepSpeedPlugin. + + If not using deepspeed, returns `None`. + """ + # To maintain original behavior, return None if not using deepspeed. + if self.distributed_type != DistributedType.DEEPSPEED: + return None + from accelerate.utils.deepspeed import get_active_deepspeed_plugin + + return get_active_deepspeed_plugin(self) + + @deepspeed_required + def get_deepspeed_plugin(self, name: str): + """ + Returns the DeepSpeedPlugin with the given plugin_key. + """ + return self.deepspeed_plugins[name] + + @deepspeed_required + def select_deepspeed_plugin(self, name: str | None = None): + """ + Activates the DeepSpeedPlugin with the given `name`, and will disable all other plugins. + """ + for key, plugin in self.deepspeed_plugins.items(): + if key != name: + plugin._unselect() + self.deepspeed_plugins[name].select(_from_accelerator_state=True) + + def print(self, *args, **kwargs): + PartialState().print(*args, **kwargs) + + def __getattr__(self, name: str): + # By this point we know that no attributes of `self` contain `name`, + # so we just modify the error message + if name in self._known_attrs: + raise AttributeError( + f"`AcceleratorState` object has no attribute `{name}`. " + "This happens if `AcceleratorState._reset_state()` was called and " + "an `Accelerator` or `PartialState` was not reinitialized." + ) + # Raise a typical AttributeError + raise AttributeError(f"'AcceleratorState' object has no attribute '{name}'") + + +class GradientState: + """ + Singleton class that has information related to gradient synchronization for gradient accumulation + + **Available attributes:** + + - **end_of_dataloader** (`bool`) -- Whether we have reached the end the current dataloader + - **remainder** (`int`) -- The number of extra samples that were added from padding the dataloader + - **sync_gradients** (`bool`) -- Whether the gradients should be synced across all devices + - **active_dataloader** (`Optional[DataLoader]`) -- The dataloader that is currently being iterated over + - **dataloader_references** (`List[Optional[DataLoader]]`) -- A list of references to the dataloaders that are + being iterated over + - **num_steps** (`int`) -- The number of steps to accumulate over + - **adjust_scheduler** (`bool`) -- Whether the scheduler should be adjusted to account for the gradient + accumulation + - **sync_with_dataloader** (`bool`) -- Whether the gradients should be synced at the end of the dataloader + iteration and the number of total steps reset + - **is_xla_gradients_synced** (`bool`) -- Whether the XLA gradients have been synchronized. It is initialized + as false. Once gradients have been reduced before the optimizer step, this flag is set to true. Subsequently, + after each step, the flag is reset to false. FSDP will always synchronize the gradients, hence + is_xla_gradients_synced is always true. + """ + + _shared_state = SharedDict() + + def __init__(self, gradient_accumulation_plugin: GradientAccumulationPlugin | None = None): + self.__dict__ = self._shared_state + if not self.initialized: + self.sync_gradients = True + self._dataloader_references_ref = [None] + self.plugin_kwargs = ( + gradient_accumulation_plugin.to_kwargs() if gradient_accumulation_plugin is not None else {} + ) + self._is_xla_gradients_synced = False + + # Plugin args are different and can be updated + if gradient_accumulation_plugin is not None and self.plugin_kwargs != gradient_accumulation_plugin.to_kwargs(): + self.plugin_kwargs = gradient_accumulation_plugin.to_kwargs() + + @property + def num_steps(self) -> int: + "Returns the number of steps to accumulate over" + return self.plugin_kwargs.get("num_steps", 1) + + @property + def adjust_scheduler(self) -> bool: + "Returns whether the scheduler should be adjusted" + return self.plugin_kwargs.get("adjust_scheduler", False) + + @property + def sync_with_dataloader(self) -> bool: + "Returns whether the gradients should be synced at the end of the dataloader iteration and the number of total steps reset" + return self.plugin_kwargs.get("sync_with_dataloader", True) + + @property + def initialized(self) -> bool: + "Returns whether the `GradientState` has been initialized" + return GradientState._shared_state != {} + + @property + def end_of_dataloader(self) -> bool: + "Returns whether we have reached the end of the current dataloader" + if not self.in_dataloader: + return False + return self.active_dataloader.end_of_dataloader + + @property + def remainder(self) -> int: + "Returns the number of extra samples that were added from padding the dataloader" + if not self.in_dataloader: + return -1 + return self.active_dataloader.remainder + + def __repr__(self): + return ( + f"Sync Gradients: {self.sync_gradients}\n" + f"At end of current dataloader: {self.end_of_dataloader}\n" + f"Extra samples added: {self.remainder}\n" + f"Gradient accumulation plugin: {self.plugin_kwargs}\n" + ) + + @property + def is_xla_gradients_synced(self): + "Returns the value of is_xla_gradients_synced. FSDP will always synchronize the gradients, hence is_xla_gradients_synced is always true." + if parse_flag_from_env("ACCELERATE_USE_FSDP", default=False): + return True + return self._is_xla_gradients_synced + + @is_xla_gradients_synced.setter + def is_xla_gradients_synced(self, is_synced): + "Set the _is_xla_gradients_synced attribute." + self._is_xla_gradients_synced = is_synced + + def _set_sync_gradients(self, sync_gradients): + "Private function that sets whether gradients should be synchronized. Users should not have to call this." + self.sync_gradients = sync_gradients + # Allow grad-sync to automatically work on TPUs + if ( + self.sync_gradients + and is_torch_xla_available(check_is_tpu=True) + and PartialState().distributed_type == DistributedType.XLA + ): + xm.mark_step() + + def _add_dataloader(self, dataloader): + "Private function that adds a dataloader to `self.dataloader_references` and sets `in_dataloader` to `True`. Users should not have to call this." + # We explicitly use assignment to ensure that the property setter is triggered, which is required for garbage collection. + # Avoid using self.dataloader_references.append as it will not trigger the setter. + self.dataloader_references += [dataloader] + + def _remove_dataloader(self, dataloader): + "Private function that removes a dataloader from `self.dataloader_references` and sets `in_dataloader` to `False` if there are no more dataloaders. Users should not have to call this." + # We explicitly use assignment to ensure that the property setter is triggered. + self.dataloader_references = [ + dataloader_ref for dataloader_ref in self.dataloader_references if dataloader_ref != dataloader + ] + + @property + def active_dataloader(self): + return self.dataloader_references[-1] + + @property + def dataloader_references(self): + # We use a property getter and setter with weakrefs to avoid circular references that prevent garbage collection + return [reference() if reference is not None else reference for reference in self._dataloader_references_ref] + + @dataloader_references.setter + def dataloader_references(self, references): + self._dataloader_references_ref = [ + weakref.ref(dataloader) if dataloader is not None else dataloader for dataloader in references + ] + + @property + def in_dataloader(self) -> bool: + "Returns whether the current process is in a dataloader" + return self.active_dataloader is not None + + @staticmethod + def _reset_state(): + "Resets `_shared_state`, is used internally and should not be called" + GradientState._shared_state.clear() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/__init__.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..67031886219f56344b1173c792ac4d8d421746d3 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/__init__.py @@ -0,0 +1,66 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .testing import ( + DEFAULT_LAUNCH_COMMAND, + are_the_same_tensors, + assert_exception, + capture_call_output, + device_count, + execute_subprocess_async, + get_launch_command, + get_torch_dist_unique_port, + memory_allocated_func, + path_in_accelerate_package, + pytest_xdist_worker_id, + require_bnb, + require_cpu, + require_cuda, + require_cuda_or_hpu, + require_cuda_or_xpu, + require_fp8, + require_fp16, + require_huggingface_suite, + require_mlu, + require_mps, + require_multi_device, + require_multi_gpu, + require_multi_gpu_or_xpu, + require_multi_xpu, + require_musa, + require_non_cpu, + require_non_hpu, + require_non_torch_xla, + require_non_xpu, + require_npu, + require_pippy, + require_sdaa, + require_single_device, + require_single_gpu, + require_single_xpu, + require_torch_min_version, + require_torchao, + require_torchvision, + require_tpu, + require_transformer_engine, + require_transformer_engine_mxfp8, + require_xpu, + run_first, + skip, + slow, + torch_device, +) +from .training import RegressionDataset, RegressionModel + + +from .scripts import test_script, test_sync, test_ops # isort: skip diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c55c0eb11c661e45f14a0898c5594d1750c1471 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/examples.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/examples.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a416ab973edec221a89a914b2c207dc5384dc83b Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/examples.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/testing.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/testing.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd17c07d7b0eeab37799720e6ce90f82bd18f322 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/testing.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/training.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/training.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24e6796815a6750aec2710a5421f2ada09aab279 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/__pycache__/training.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/examples.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/examples.py new file mode 100644 index 0000000000000000000000000000000000000000..18549cdb31492650e707b6008e25eeac8e33b04e --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/examples.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python + +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +A collection of utilities for comparing `examples/complete_*_example.py` scripts with the capabilities inside of each +`examples/by_feature` example. `compare_against_test` is the main function that should be used when testing, while the +others are used to either get the code that matters, or to preprocess them (such as stripping comments) +""" + +import os +from typing import Optional + + +def get_function_contents_by_name(lines: list[str], name: str): + """ + Extracts a function from `lines` of segmented source code with the name `name`. + + Args: + lines (`List[str]`): + Source code of a script separated by line. + name (`str`): + The name of the function to extract. Should be either `training_function` or `main` + """ + if name != "training_function" and name != "main": + raise ValueError(f"Incorrect function name passed: {name}, choose either 'main' or 'training_function'") + good_lines, found_start = [], False + for line in lines: + if not found_start and f"def {name}" in line: + found_start = True + good_lines.append(line) + continue + if found_start: + if name == "training_function" and "def main" in line: + return good_lines + if name == "main" and "if __name__" in line: + return good_lines + good_lines.append(line) + + +def clean_lines(lines: list[str]): + """ + Filters `lines` and removes any entries that start with a comment ('#') or is just a newline ('\n') + + Args: + lines (`List[str]`): + Source code of a script separated by line. + """ + return [line for line in lines if not line.lstrip().startswith("#") and line != "\n"] + + +def compare_against_test( + base_filename: str, feature_filename: str, parser_only: bool, secondary_filename: Optional[str] = None +): + """ + Tests whether the additional code inside of `feature_filename` was implemented in `base_filename`. This should be + used when testing to see if `complete_*_.py` examples have all of the implementations from each of the + `examples/by_feature/*` scripts. + + It utilizes `nlp_example.py` to extract out all of the repeated training code, so that only the new additional code + is examined and checked. If something *other* than `nlp_example.py` should be used, such as `cv_example.py` for the + `complete_cv_example.py` script, it should be passed in for the `secondary_filename` parameter. + + Args: + base_filename (`str` or `os.PathLike`): + The filepath of a single "complete" example script to test, such as `examples/complete_cv_example.py` + feature_filename (`str` or `os.PathLike`): + The filepath of a single feature example script. The contents of this script are checked to see if they + exist in `base_filename` + parser_only (`bool`): + Whether to compare only the `main()` sections in both files, or to compare the contents of + `training_loop()` + secondary_filename (`str`, *optional*): + A potential secondary filepath that should be included in the check. This function extracts the base + functionalities off of "examples/nlp_example.py", so if `base_filename` is a script other than + `complete_nlp_example.py`, the template script should be included here. Such as `examples/cv_example.py` + """ + with open(base_filename) as f: + base_file_contents = f.readlines() + with open(os.path.abspath(os.path.join("examples", "nlp_example.py"))) as f: + full_file_contents = f.readlines() + with open(feature_filename) as f: + feature_file_contents = f.readlines() + if secondary_filename is not None: + with open(secondary_filename) as f: + secondary_file_contents = f.readlines() + + # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content + if parser_only: + base_file_func = clean_lines(get_function_contents_by_name(base_file_contents, "main")) + full_file_func = clean_lines(get_function_contents_by_name(full_file_contents, "main")) + feature_file_func = clean_lines(get_function_contents_by_name(feature_file_contents, "main")) + if secondary_filename is not None: + secondary_file_func = clean_lines(get_function_contents_by_name(secondary_file_contents, "main")) + else: + base_file_func = clean_lines(get_function_contents_by_name(base_file_contents, "training_function")) + full_file_func = clean_lines(get_function_contents_by_name(full_file_contents, "training_function")) + feature_file_func = clean_lines(get_function_contents_by_name(feature_file_contents, "training_function")) + if secondary_filename is not None: + secondary_file_func = clean_lines( + get_function_contents_by_name(secondary_file_contents, "training_function") + ) + + _dl_line = "train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n" + + # Specific code in our script that differs from the full version, aka what is new + new_feature_code = [] + passed_idxs = [] # We keep track of the idxs just in case it's a repeated statement + it = iter(feature_file_func) + for i in range(len(feature_file_func) - 1): + if i not in passed_idxs: + line = next(it) + if (line not in full_file_func) and (line.lstrip() != _dl_line): + if "TESTING_MOCKED_DATALOADERS" not in line: + new_feature_code.append(line) + passed_idxs.append(i) + else: + # Skip over the `config['num_epochs'] = 2` statement + _ = next(it) + + # Extract out just the new parts from the full_file_training_func + new_full_example_parts = [] + passed_idxs = [] # We keep track of the idxs just in case it's a repeated statement + for i, line in enumerate(base_file_func): + if i not in passed_idxs: + if (line not in full_file_func) and (line.lstrip() != _dl_line): + if "TESTING_MOCKED_DATALOADERS" not in line: + new_full_example_parts.append(line) + passed_idxs.append(i) + + # Finally, get the overall diff + diff_from_example = [line for line in new_feature_code if line not in new_full_example_parts] + if secondary_filename is not None: + diff_from_two = [line for line in full_file_contents if line not in secondary_file_func] + diff_from_example = [line for line in diff_from_example if line not in diff_from_two] + + return diff_from_example diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__init__.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c9cbe26c257b515f657c05e1996d517e69613972 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..73bc247d87bfc6ab0f9d4833b27c3562dd036432 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_cli.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_cli.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd57c8004cf39a77a1eb5427595bf58c8851cefd Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_cli.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_ddp_comm_hook.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_ddp_comm_hook.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e683feee9c294b2b484dbb7b599cc8bf9ec6f185 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_ddp_comm_hook.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_distributed_data_loop.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_distributed_data_loop.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5134faafbfa747a9ecd4ed0a4db68b82a54a46a4 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_distributed_data_loop.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_merge_weights.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_merge_weights.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5d8ff04f3999d3d3c5738135eee7b1247a5b50dd Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_merge_weights.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_notebook.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_notebook.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..862ed917e6c93f1d419b8a2ccdb8098e5ba7d7d7 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_notebook.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_ops.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_ops.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce57f770ae846e02149fff17cdca5e1faa50523e Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_ops.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_script.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_script.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..117646094fd5b272c769134ccef992b8cb2dd529 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_script.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_sync.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_sync.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c23df436d5184b4512220821dd66130d378dd0a8 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/__pycache__/test_sync.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__init__.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c9cbe26c257b515f657c05e1996d517e69613972 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba1a2e280ed2a1a8aa7fabce54f652a968d70c10 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_checkpointing.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_checkpointing.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..03cbe47d5a31cfc19ada3529c9baca68739ffaf0 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_checkpointing.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_ds_alst_ulysses_sp.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_ds_alst_ulysses_sp.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0dc9fd0368e12506f0d6860e502190fd9d7c922a Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_ds_alst_ulysses_sp.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_ds_multiple_model.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_ds_multiple_model.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66c77fc45a958e3b6553f1a167aa111084eb61f9 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_ds_multiple_model.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_metrics.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_metrics.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b0bd0c393f7fc51809e1a538e3cb0925d0c98c05 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_metrics.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_peak_memory_usage.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_peak_memory_usage.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5fbb4670343521387c17534c6343372b40e99f08 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_peak_memory_usage.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_performance.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_performance.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..124990e2acd58f324fae42118deddda4032b7fa4 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_performance.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_pippy.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_pippy.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ca29b49d1e38127fa4fc4f20dc4dea995f71dd6 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_pippy.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_zero3_integration.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_zero3_integration.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9ff37e4b7ddcd51d6ab94d2933639a53311852d5 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/__pycache__/test_zero3_integration.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_checkpointing.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_checkpointing.py new file mode 100644 index 0000000000000000000000000000000000000000..e84d77dc8df4a1bf95d7e4b2d2bb3c80c945b259 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_checkpointing.py @@ -0,0 +1,269 @@ +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import os + +import evaluate +import torch +from datasets import load_dataset +from torch.optim import AdamW +from torch.utils.data import DataLoader +from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed + +from accelerate import Accelerator, DistributedType +from accelerate.utils.deepspeed import DummyOptim, DummyScheduler + + +MAX_GPU_BATCH_SIZE = 16 +EVAL_BATCH_SIZE = 32 + + +def get_dataloaders(accelerator: Accelerator, batch_size: int = 16, model_name: str = "bert-base-cased"): + """ + Creates a set of `DataLoader`s for the `glue` dataset. + + Args: + accelerator (`Accelerator`): + An `Accelerator` object + batch_size (`int`, *optional*): + The batch size for the train and validation DataLoaders. + model_name (`str`, *optional*): + """ + tokenizer = AutoTokenizer.from_pretrained(model_name) + datasets = load_dataset("nyu-mll/glue", "mrpc") + + def tokenize_function(examples): + # max_length=None => use the model max length (it's actually the default) + outputs = tokenizer(examples["sentence1"], examples["sentence2"], truncation=True, max_length=None) + return outputs + + # Apply the method we just defined to all the examples in all the splits of the dataset + tokenized_datasets = datasets.map( + tokenize_function, batched=True, remove_columns=["idx", "sentence1", "sentence2"], load_from_cache_file=False + ) + + # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the + # transformers library + tokenized_datasets = tokenized_datasets.rename_column("label", "labels") + + def collate_fn(examples): + # On TPU it's best to pad everything to the same length or training will be very slow. + if accelerator.distributed_type == DistributedType.XLA: + return tokenizer.pad(examples, padding="max_length", max_length=128, return_tensors="pt") + return tokenizer.pad(examples, padding="longest", return_tensors="pt") + + # Instantiate dataloaders. + train_dataloader = DataLoader( + tokenized_datasets["train"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size + ) + eval_dataloader = DataLoader( + tokenized_datasets["validation"], shuffle=False, collate_fn=collate_fn, batch_size=EVAL_BATCH_SIZE + ) + + return train_dataloader, eval_dataloader + + +def evaluation_loop(accelerator, model, eval_dataloader, metric): + model.eval() + samples_seen = 0 + for step, batch in enumerate(eval_dataloader): + # We could avoid this line since we set the accelerator with `device_placement=True`. + batch.to(accelerator.device) + with torch.no_grad(): + outputs = model(**batch) + predictions = outputs.logits.argmax(dim=-1) + # It is slightly faster to call this once, than multiple times + predictions, references = accelerator.gather( + (predictions, batch["labels"]) + ) # If we are in a multiprocess environment, the last batch has duplicates + if accelerator.use_distributed: + if step == len(eval_dataloader) - 1: + predictions = predictions[: len(eval_dataloader.dataset) - samples_seen] + references = references[: len(eval_dataloader.dataset) - samples_seen] + else: + samples_seen += references.shape[0] + metric.add_batch( + predictions=predictions, + references=references, + ) + + eval_metric = metric.compute() + return eval_metric["accuracy"] + + +def training_function(config, args): + # Initialize accelerator + accelerator = Accelerator() + + # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs + lr = config["lr"] + num_epochs = int(config["num_epochs"]) + seed = int(config["seed"]) + batch_size = int(config["batch_size"]) + model_name = args.model_name_or_path + + set_seed(seed) + train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size, model_name) + + # Instantiate the model (we build the model here so that the seed also control new weights initialization) + model = AutoModelForSequenceClassification.from_pretrained(model_name, return_dict=True) + + # Instantiate optimizer + optimizer_cls = ( + AdamW + if accelerator.state.deepspeed_plugin is None + or "optimizer" not in accelerator.state.deepspeed_plugin.deepspeed_config + else DummyOptim + ) + optimizer = optimizer_cls(params=model.parameters(), lr=lr) + + if accelerator.state.deepspeed_plugin is not None: + gradient_accumulation_steps = accelerator.state.deepspeed_plugin.deepspeed_config[ + "gradient_accumulation_steps" + ] + else: + gradient_accumulation_steps = 1 + max_training_steps = (len(train_dataloader) * num_epochs) // gradient_accumulation_steps + + # Instantiate scheduler + if ( + accelerator.state.deepspeed_plugin is None + or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config + ): + lr_scheduler = get_linear_schedule_with_warmup( + optimizer=optimizer, + num_warmup_steps=0, + num_training_steps=max_training_steps, + ) + else: + lr_scheduler = DummyScheduler(optimizer, total_num_steps=max_training_steps, warmup_num_steps=0) + + # Prepare everything + # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the + # prepare method. + model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare( + model, optimizer, train_dataloader, eval_dataloader, lr_scheduler + ) + + # We need to keep track of how many total steps we have iterated over + overall_step = 0 + # We also need to keep track of the stating epoch so files are named properly + starting_epoch = 0 + metric = evaluate.load("glue", "mrpc") + ending_epoch = num_epochs + + if args.partial_train_epoch is not None: + ending_epoch = args.partial_train_epoch + + if args.resume_from_checkpoint: + accelerator.load_state(args.resume_from_checkpoint) + epoch_string = args.resume_from_checkpoint.split("epoch_")[1] + state_epoch_num = "" + for char in epoch_string: + if char.isdigit(): + state_epoch_num += char + else: + break + starting_epoch = int(state_epoch_num) + 1 + accuracy = evaluation_loop(accelerator, model, eval_dataloader, metric) + accelerator.print("resumed checkpoint performance:", accuracy) + accelerator.print("resumed checkpoint's scheduler's lr:", lr_scheduler.get_lr()[0]) + accelerator.print("resumed optimizers's lr:", optimizer.param_groups[0]["lr"]) + with open(os.path.join(args.output_dir, f"state_{starting_epoch - 1}.json")) as f: + resumed_state = json.load(f) + assert resumed_state["accuracy"] == accuracy, "Accuracy mismatch, loading from checkpoint failed" + assert resumed_state["lr"] == lr_scheduler.get_lr()[0], ( + "Scheduler learning rate mismatch, loading from checkpoint failed" + ) + assert resumed_state["optimizer_lr"] == optimizer.param_groups[0]["lr"], ( + "Optimizer learning rate mismatch, loading from checkpoint failed" + ) + assert resumed_state["epoch"] == starting_epoch - 1, "Epoch mismatch, loading from checkpoint failed" + return + + # Now we train the model + state = {} + for epoch in range(starting_epoch, ending_epoch): + model.train() + for step, batch in enumerate(train_dataloader): + outputs = model(**batch) + loss = outputs.loss + loss = loss / gradient_accumulation_steps + accelerator.backward(loss) + if step % gradient_accumulation_steps == 0: + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + overall_step += 1 + output_dir = f"epoch_{epoch}" + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) + accuracy = evaluation_loop(accelerator, model, eval_dataloader, metric) + state["accuracy"] = accuracy + state["lr"] = lr_scheduler.get_lr()[0] + state["optimizer_lr"] = optimizer.param_groups[0]["lr"] + state["epoch"] = epoch + state["step"] = overall_step + accelerator.print(f"epoch {epoch}:", state) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + with open(os.path.join(args.output_dir, f"state_{epoch}.json"), "w") as f: + json.dump(state, f) + accelerator.end_training() + + +def main(): + parser = argparse.ArgumentParser(description="Simple example of training script tracking peak GPU memory usage.") + parser.add_argument( + "--model_name_or_path", + type=str, + default="bert-base-cased", + help="Path to pretrained model or model identifier from huggingface.co/models.", + required=False, + ) + parser.add_argument( + "--output_dir", + type=str, + default=".", + help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory.", + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help="If the training should continue from a checkpoint folder.", + ) + parser.add_argument( + "--partial_train_epoch", + type=int, + default=None, + help="If passed, the training will stop after this number of epochs.", + ) + parser.add_argument( + "--num_epochs", + type=int, + default=2, + help="Number of train epochs.", + ) + args = parser.parse_args() + config = {"lr": 2e-5, "num_epochs": args.num_epochs, "seed": 42, "batch_size": 16} + + training_function(config, args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_ds_alst_ulysses_sp.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_ds_alst_ulysses_sp.py new file mode 100644 index 0000000000000000000000000000000000000000..c756d5dc77f9ce51e305001697fad5c342575887 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_ds_alst_ulysses_sp.py @@ -0,0 +1,131 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test script for verifying ALST/Ulysses SP works +""" + +import torch +from deepspeed.runtime.utils import move_to_device +from transformers import AutoModelForCausalLM, AutoTokenizer + +from accelerate import Accelerator +from accelerate.utils import ParallelismConfig, set_seed +from accelerate.utils.dataclasses import DeepSpeedSequenceParallelConfig + + +set_seed(42) + +world_size = 2 +model_name = "hf-internal-testing/tiny-random-LlamaForCausalLM" + +micro_batch_size = 1 + +parallelism_config = ParallelismConfig( + sp_backend="deepspeed", + sp_size=world_size, + # dp_shard_size=1, # set if dp is wanted as well + sp_handler=DeepSpeedSequenceParallelConfig( + sp_seq_length=256, + sp_seq_length_is_variable=True, + sp_attn_implementation="sdpa", + ), +) + +accelerator = Accelerator( + parallelism_config=parallelism_config, +) + +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained(model_name) + +samples = 4 +seqlen = 32 +input_ids = torch.arange(1, seqlen * samples + 1).view(-1, seqlen) + 100 +position_ids = torch.arange(seqlen * samples).view(-1, seqlen) + +ds = torch.utils.data.TensorDataset(input_ids, position_ids) + + +def collate_fn(batch): + input_ids, position_ids = batch[0] + return dict( + input_ids=input_ids.unsqueeze(0), + position_ids=position_ids.unsqueeze(0), + labels=input_ids.unsqueeze(0), + ) + + +dl = torch.utils.data.DataLoader(ds, batch_size=micro_batch_size, collate_fn=collate_fn) + +optimizer = torch.optim.Adam(model.parameters(), lr=1e-5) + +rank = torch.distributed.get_rank() + +if rank == 0: + print(f"DL orig: {len(dl)} samples") + +model, optimizer, dl = accelerator.prepare(model, optimizer, dl) + +if rank == 0: + print(f"DL w/ adapter: {len(dl)} samples") + +sp_size = parallelism_config.sp_size if parallelism_config else 1 +if sp_size > 1: + from deepspeed.utils import groups + + sp_group = groups._get_sequence_parallel_group() + sp_world_size = parallelism_config.sp_size + +unwrapped_model = accelerator.unwrap_model(model) + +# Normal training loop +for iter, batch in enumerate(dl): + optimizer.zero_grad() + + if rank == 0: + print(f"batch {iter}: seqlen: {len(batch['input_ids'][0])}") + batch = move_to_device(batch, model.device) + outputs = model(**batch) + + shift_labels = batch["shift_labels"] + loss = unwrapped_model.loss_function( + logits=outputs.logits, + labels=None, + shift_labels=shift_labels, + vocab_size=unwrapped_model.config.vocab_size, + ) + + if sp_size > 1: + # differentiable weighted per-shard-loss aggregation across ranks + losses_per_rank = torch.distributed.nn.functional.all_gather(loss, group=sp_group) + # special dealing with SFT that has prompt tokens that aren't used in loss computation + good_tokens = (shift_labels != -100).view(-1).sum() + good_tokens_per_rank = torch.distributed.nn.functional.all_gather(good_tokens, group=sp_group) + total_loss = sum( + losses_per_rank[rank] * good_tokens_per_rank[rank] + for rank in range(sp_world_size) + if good_tokens_per_rank[rank] > 0 + ) + total_good_tokens = sum(good_tokens_per_rank) + loss = total_loss / max(total_good_tokens, 1) + + if rank == 0: + accelerator.print(f"{iter}: {loss=}") + accelerator.log(dict(train_loss=loss, step=iter)) + + accelerator.backward(loss) + optimizer.step() + +accelerator.end_training() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_ds_multiple_model.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_ds_multiple_model.py new file mode 100644 index 0000000000000000000000000000000000000000..8fa8b5552bd8e69f9f0fd9036644c5d109a2fb0d --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_ds_multiple_model.py @@ -0,0 +1,331 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test script for verifying multiple models can be utilized with Accelerate + DeepSpeed: + +Scenario 1: One model is training, another model is being used for inference/logits to impact training in some form. +Scenario 2: Two models are training simultaneously, which means two optimizers, etc. +""" + +import argparse +from pathlib import Path + +import evaluate +import torch +from datasets import load_dataset +from torch.optim import AdamW +from torch.utils.data import DataLoader +from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup + +from accelerate import Accelerator, DeepSpeedPlugin, DistributedType +from accelerate.state import AcceleratorState +from accelerate.utils.deepspeed import get_active_deepspeed_plugin + + +EVAL_BATCH_SIZE = 16 + + +class NoiseModel(torch.nn.Module): + def __init__(self, noise_factor=0.1): + super().__init__() + self.noise_factor = torch.nn.Parameter(torch.tensor(noise_factor, dtype=torch.float32)) + + def forward(self, loss): + return loss * self.noise_factor + + +def get_dataloaders(accelerator: Accelerator, batch_size: int = 16, model_name: str = "bert-base-cased"): + """ + Creates a set of `DataLoader`s for the `glue` dataset. + + Args: + accelerator (`Accelerator`): + An `Accelerator` object + batch_size (`int`, *optional*): + The batch size for the train and validation DataLoaders. + model_name (`str`, *optional*): + """ + tokenizer = AutoTokenizer.from_pretrained(model_name) + datasets = load_dataset("nyu-mll/glue", "mrpc") + + def tokenize_function(examples): + # max_length=None => use the model max length (it's actually the default) + outputs = tokenizer(examples["sentence1"], examples["sentence2"], truncation=True, max_length=None) + return outputs + + # Apply the method we just defined to all the examples in all the splits of the dataset + tokenized_datasets = datasets.map( + tokenize_function, batched=True, remove_columns=["idx", "sentence1", "sentence2"], load_from_cache_file=False + ) + + # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the + # transformers library + tokenized_datasets = tokenized_datasets.rename_column("label", "labels") + + def collate_fn(examples): + # On TPU it's best to pad everything to the same length or training will be very slow. + if accelerator.distributed_type == DistributedType.XLA: + return tokenizer.pad(examples, padding="max_length", max_length=128, return_tensors="pt") + return tokenizer.pad(examples, padding="longest", return_tensors="pt") + + # Instantiate dataloaders. + train_dataloader = DataLoader( + tokenized_datasets["train"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size + ) + eval_dataloader = DataLoader( + tokenized_datasets["validation"], shuffle=False, collate_fn=collate_fn, batch_size=EVAL_BATCH_SIZE + ) + + return train_dataloader, eval_dataloader + + +test_file_path = __file__ +path = Path(test_file_path).resolve() +test_file_dir_str = str(path.parent.parent.parent.parent.parent.parent) + +# Create our DS plugins +# We use custom schedulers and optimizers, hence `model_only` +ds_config_file = dict( + zero2=f"{test_file_dir_str}/tests/deepspeed/ds_config_zero2_model_only.json", + zero3=f"{test_file_dir_str}/tests/deepspeed/ds_config_zero3_model_only.json", +) + + +def single_model_training(config, args): + # Training a single model, we have a `noise` model that is untrainable used to inject some noise into the training process + num_epochs = config["num_epochs"] + zero2_plugin = DeepSpeedPlugin(hf_ds_config=ds_config_file["zero2"]) + zero3_plugin = DeepSpeedPlugin(hf_ds_config=ds_config_file["zero3"]) + + deepspeed_plugins = {"training": zero2_plugin, "inference": zero3_plugin} + + # Initialize accelerator + accelerator = Accelerator( + deepspeed_plugins=deepspeed_plugins, + mixed_precision="bf16", + ) + + # Initialize model under zero2 plugin + assert get_active_deepspeed_plugin(accelerator.state) is zero2_plugin + train_model = AutoModelForSequenceClassification.from_pretrained(args.model_name_or_path) + train_dataloader, eval_dataloader = get_dataloaders( + accelerator, batch_size=config["batch_size"], model_name=args.model_name_or_path + ) + max_training_steps = len(train_dataloader) * config["num_epochs"] + optimizer = AdamW(train_model.parameters(), lr=config["lr"]) + lr_scheduler = get_linear_schedule_with_warmup( + optimizer, num_warmup_steps=0, num_training_steps=max_training_steps + ) + + train_dataloader, eval_dataloader, train_model, optimizer, lr_scheduler = accelerator.prepare( + train_dataloader, eval_dataloader, train_model, optimizer, lr_scheduler + ) + + # Now prepare the model under zero3 plugin + accelerator.state.select_deepspeed_plugin("inference") + assert get_active_deepspeed_plugin(accelerator.state) is zero3_plugin + inference_model = NoiseModel() + inference_model = accelerator.prepare(inference_model) + inference_model.eval() + + # Run training loop + accelerator.state.select_deepspeed_plugin("training") + # We also need to keep track of the stating epoch so files are named properly + starting_epoch = 0 + + # Now we train the model + best_performance = 0 + metric = evaluate.load("glue", "mrpc") + performance_metric = {} + for epoch in range(starting_epoch, num_epochs): + train_model.train() + inference_model.train() + for step, batch in enumerate(train_dataloader): + with accelerator.accumulate(train_model): + outputs_1 = train_model(**batch) + with torch.no_grad(): + outputs_2 = inference_model(outputs_1.loss) + # Combine the losses + loss = outputs_1.loss + outputs_2 + accelerator.backward(loss) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + train_model.eval() + for step, batch in enumerate(eval_dataloader): + with torch.no_grad(): + outputs = train_model(**batch) + predictions = outputs.logits.argmax(dim=-1) + # It is slightly faster to call this once, than multiple times + predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"])) + metric.add_batch( + predictions=predictions, + references=references, + ) + + eval_metric = metric.compute() + # Use accelerator.print to print only on the main process. + accelerator.print(f"epoch {epoch}:", eval_metric) + performance_metric[f"epoch-{epoch}"] = eval_metric["accuracy"] + + if best_performance < eval_metric["accuracy"]: + best_performance = eval_metric["accuracy"] + assert best_performance > performance_metric["epoch-0"] + + +def multiple_model_training(config, args): + # This will essentially be like a k-fold model, but one model is Zero-2 and another model is Zero-3 + num_epochs = config["num_epochs"] + zero2_plugin = DeepSpeedPlugin(hf_ds_config=ds_config_file["zero2"]) + zero3_plugin = DeepSpeedPlugin(hf_ds_config=ds_config_file["zero3"]) + + deepspeed_plugins = {"zero2": zero2_plugin, "zero3": zero3_plugin} + + # Initialize accelerator + zero2_accelerator = Accelerator( + deepspeed_plugins=deepspeed_plugins, + mixed_precision="bf16", + ) + + # Since an `AcceleratorState` has already been made, we can just reuse it here + zero3_accelerator = Accelerator() + + # Initialize model under zero2 plugin + assert get_active_deepspeed_plugin(zero2_accelerator.state) is zero2_plugin + zero2_model = AutoModelForSequenceClassification.from_pretrained(args.model_name_or_path) + train_dataloader, eval_dataloader = get_dataloaders( + zero2_accelerator, batch_size=config["batch_size"], model_name=args.model_name_or_path + ) + max_training_steps = len(train_dataloader) * config["num_epochs"] + zero2_optimizer = AdamW(zero2_model.parameters(), lr=config["lr"]) + zero2_lr_scheduler = get_linear_schedule_with_warmup( + zero2_optimizer, num_warmup_steps=0, num_training_steps=max_training_steps + ) + + train_dataloader, eval_dataloader, zero2_model, zero2_optimizer, zero2_lr_scheduler = zero2_accelerator.prepare( + train_dataloader, eval_dataloader, zero2_model, zero2_optimizer, zero2_lr_scheduler + ) + assert zero2_accelerator.deepspeed_engine_wrapped.engine is zero2_model + + # now do Zero3 + zero3_accelerator.state.select_deepspeed_plugin("zero3") + zero3_plugin.deepspeed_config["train_micro_batch_size_per_gpu"] = zero2_plugin.deepspeed_config[ + "train_micro_batch_size_per_gpu" + ] + assert get_active_deepspeed_plugin(zero3_accelerator.state) is zero3_plugin + zero3_model = AutoModelForSequenceClassification.from_pretrained(args.model_name_or_path) + zero3_optimizer = AdamW(zero3_model.parameters(), lr=config["lr"]) + zero3_lr_scheduler = get_linear_schedule_with_warmup( + zero3_optimizer, num_warmup_steps=0, num_training_steps=max_training_steps + ) + zero3_model, zero3_optimizer, zero3_lr_scheduler = zero3_accelerator.prepare( + zero3_model, zero3_optimizer, zero3_lr_scheduler + ) + assert zero3_accelerator.deepspeed_engine_wrapped.engine is zero3_model + + # Run training loop + starting_epoch = 0 + + # Now we train the model + best_performance_a = 0 + best_performance_b = 0 + metric_a = evaluate.load("glue", "mrpc") + metric_b = evaluate.load("glue", "mrpc") + performance_metric_a = {} + performance_metric_b = {} + for epoch in range(starting_epoch, num_epochs): + zero2_model.train() + zero3_model.train() + for step, batch in enumerate(train_dataloader): + with zero2_accelerator.accumulate(zero2_model, zero3_model): + outputs_1 = zero2_model(**batch) + zero2_accelerator.backward(outputs_1.loss) + zero2_optimizer.step() + zero2_lr_scheduler.step() + zero2_optimizer.zero_grad() + outputs_2 = zero3_model(**batch) + zero3_accelerator.backward(outputs_2.loss) + zero3_optimizer.step() + zero3_lr_scheduler.step() + zero3_optimizer.zero_grad() + + zero2_model.eval() + zero3_model.eval() + for step, batch in enumerate(eval_dataloader): + with torch.no_grad(): + logits_a = zero2_model(**batch).logits + logits_b = zero3_model(**batch).logits + # Combine the logits from both models + predictions_a = logits_a.argmax(dim=-1) + predictions_b = logits_b.argmax(dim=-1) + # It is slightly faster to call this once, than multiple times + predictions_a, predictions_b, references = zero2_accelerator.gather_for_metrics( + (predictions_a, predictions_b, batch["labels"]) + ) + metric_a.add_batch( + predictions=predictions_a, + references=references, + ) + metric_b.add_batch( + predictions=predictions_b, + references=references, + ) + + eval_metric_a = metric_a.compute() + eval_metric_b = metric_b.compute() + # Use accelerator.print to print only on the main process. + zero2_accelerator.print(f"epoch {epoch}:", eval_metric_a, eval_metric_b) + performance_metric_a[f"epoch-{epoch}"] = eval_metric_a["accuracy"] + performance_metric_b[f"epoch-{epoch}"] = eval_metric_b["accuracy"] + + if best_performance_a < eval_metric_a["accuracy"]: + best_performance_a = eval_metric_a["accuracy"] + if best_performance_b < eval_metric_b["accuracy"]: + best_performance_b = eval_metric_b["accuracy"] + assert best_performance_a > performance_metric_a["epoch-0"] + assert best_performance_b > performance_metric_b["epoch-0"] + + +def main(): + parser = argparse.ArgumentParser(description="Simple example of training script tracking peak GPU memory usage.") + parser.add_argument( + "--model_name_or_path", + type=str, + default="bert-base-cased", + help="Path to pretrained model or model identifier from huggingface.co/models.", + required=False, + ) + parser.add_argument( + "--performance_lower_bound", + type=float, + default=None, + help="Optional lower bound for the performance metric. If set, the training will throw error when the performance metric drops below this value.", + ) + parser.add_argument( + "--num_epochs", + type=int, + default=3, + help="Number of train epochs.", + ) + args = parser.parse_args() + config = {"lr": 2e-5, "num_epochs": args.num_epochs, "seed": 42, "batch_size": 8} + single_model_training(config, args) + AcceleratorState._reset_state(True) + multiple_model_training(config, args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_metrics.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..2140f8a69ee8719db95f4f8c6a5588bdf045bb46 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_metrics.py @@ -0,0 +1,307 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import math +import os +from copy import deepcopy + +import datasets +import evaluate +import torch +import transformers +from datasets import load_dataset +from torch.utils.data import DataLoader, IterableDataset +from transformers import AutoModelForSequenceClassification, AutoTokenizer + +from accelerate import Accelerator, DataLoaderConfiguration, DistributedType +from accelerate.data_loader import DataLoaderDispatcher +from accelerate.test_utils import RegressionDataset, RegressionModel, torch_device +from accelerate.utils import is_torch_xla_available, set_seed + + +os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "true" + + +class ListHandler(logging.Handler): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.logs = [] + + def emit(self, record): + self.logs.append(record) + + +def get_basic_setup(accelerator, num_samples=82, batch_size=16): + "Returns everything needed to perform basic training" + set_seed(42) + model = RegressionModel() + ddp_model = deepcopy(model) + dset = RegressionDataset(length=num_samples) + dataloader = DataLoader(dset, batch_size=batch_size) + model.to(accelerator.device) + ddp_model, dataloader = accelerator.prepare(ddp_model, dataloader) + return model, ddp_model, dataloader + + +def get_dataloader(accelerator: Accelerator, use_longest=False): + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/mrpc-bert-base-cased") + dataset = load_dataset("nyu-mll/glue", "mrpc", split="validation") + + def tokenize_function(examples): + outputs = tokenizer(examples["sentence1"], examples["sentence2"], truncation=True, max_length=None) + return outputs + + with accelerator.main_process_first(): + tokenized_datasets = dataset.map( + tokenize_function, + batched=True, + remove_columns=["idx", "sentence1", "sentence2"], + ) + + tokenized_datasets = tokenized_datasets.rename_column("label", "labels") + + def collate_fn(examples): + if use_longest: + return tokenizer.pad(examples, padding="longest", return_tensors="pt") + return tokenizer.pad(examples, padding="max_length", max_length=128, return_tensors="pt") + + return DataLoader(tokenized_datasets, shuffle=False, collate_fn=collate_fn, batch_size=16) + + +def get_mrpc_setup(dispatch_batches, split_batches): + dataloader_config = DataLoaderConfiguration(dispatch_batches=dispatch_batches, split_batches=split_batches) + accelerator = Accelerator(dataloader_config=dataloader_config) + dataloader = get_dataloader(accelerator, not dispatch_batches) + model = AutoModelForSequenceClassification.from_pretrained( + "hf-internal-testing/mrpc-bert-base-cased", return_dict=True + ) + ddp_model, ddp_dataloader = accelerator.prepare(model, dataloader) + return { + "ddp": [ddp_model, ddp_dataloader, torch_device], + "no": [model, dataloader, accelerator.device], + }, accelerator + + +def generate_predictions(model, dataloader, accelerator): + logits_and_targets = [] + for batch in dataloader: + input, target = batch.values() + with torch.no_grad(): + logit = model(input) + logit, target = accelerator.gather_for_metrics((logit, target)) + logits_and_targets.append((logit, target)) + logits, targs = [], [] + for logit, targ in logits_and_targets: + logits.append(logit) + targs.append(targ) + logits, targs = torch.cat(logits), torch.cat(targs) + return logits, targs + + +def test_torch_metrics( + accelerator: Accelerator, num_samples=82, dispatch_batches=False, split_batches=False, batch_size=16 +): + _, ddp_model, dataloader = get_basic_setup(accelerator, num_samples, batch_size) + logits, _ = generate_predictions(ddp_model, dataloader, accelerator) + assert len(logits) == num_samples, ( + f"Unexpected number of inputs:\n Expected: {num_samples}\n Actual: {len(logits)}" + ) + + +def test_mrpc(dispatch_batches: bool = False, split_batches: bool = False): + metric = evaluate.load("glue", "mrpc") + setup, accelerator = get_mrpc_setup(dispatch_batches, split_batches) + # First do baseline + model, dataloader, device = setup["no"] + model.to(device) + model.eval() + for batch in dataloader: + batch.to(device) + with torch.inference_mode(): + outputs = model(**batch) + preds = outputs.logits.argmax(dim=-1) + metric.add_batch(predictions=preds, references=batch["labels"]) + baseline = metric.compute() + + # Then do distributed + model, dataloader, device = setup["ddp"] + model.eval() + for batch in dataloader: + with torch.inference_mode(): + outputs = model(**batch) + preds = outputs.logits.argmax(dim=-1) + references = batch["labels"] + preds, references = accelerator.gather_for_metrics((preds, references)) + metric.add_batch(predictions=preds, references=references) + distributed = metric.compute() + + for key in "accuracy f1".split(): + assert math.isclose(baseline[key], distributed[key]), ( + f"Baseline and Distributed are not the same for key {key}:\n\tBaseline: {baseline[key]}\n\tDistributed: {distributed[key]}\n" + ) + + +def test_gather_for_metrics_with_non_tensor_objects_iterable_dataset(): + class DummyIterableDataset(IterableDataset): + def __init__(self, data): + self.data = data + + def __len__(self): + return len(self.data) + + def __iter__(self): + yield from self.data + + iterable_dataset = DummyIterableDataset([n for n in range(30)]) + dataloader = DataLoader(iterable_dataset, batch_size=4) + accelerator = Accelerator() + prepared_dataloader = accelerator.prepare(dataloader) + + if accelerator.is_main_process: + logger = logging.root.manager.loggerDict["accelerate.accelerator"] + list_handler = ListHandler() + logger.addHandler(list_handler) + + batches_for_metrics = [] + for batch in prepared_dataloader: + batches_for_metrics.append(accelerator.gather_for_metrics(batch)) + + assert torch.cat(batches_for_metrics).size(0) == 30 + + if accelerator.is_main_process: + assert len(list_handler.logs) == 0 + logger.removeHandler(list_handler) + + +def test_gather_for_metrics_with_iterable_dataset(): + class DummyIterableDataset(IterableDataset): + def __init__(self, data): + self.data = data + + def __len__(self): + return len(self.data) + + def __iter__(self): + yield from self.data + + iterable_dataset = DummyIterableDataset(torch.as_tensor(range(30))) + dataloader = DataLoader(iterable_dataset, batch_size=4) + + accelerator = Accelerator() + prepared_dataloader = accelerator.prepare(dataloader) + + assert isinstance(prepared_dataloader, DataLoaderDispatcher) + + if accelerator.is_main_process: + logger = logging.root.manager.loggerDict["accelerate.accelerator"] + list_handler = ListHandler() + logger.addHandler(list_handler) + + batches_for_metrics = [] + for batch in prepared_dataloader: + batches_for_metrics.append(accelerator.gather_for_metrics(batch)) + + assert torch.cat(batches_for_metrics).size(0) == 30 + + if accelerator.is_main_process: + assert len(list_handler.logs) == 0 + + logger.removeHandler(list_handler) + + +def test_gather_for_metrics_drop_last(): + accelerator = Accelerator() + per_device_batch_size = 5 + num_items = (10 * accelerator.num_processes) + 1 + dataloader = DataLoader(range(num_items), batch_size=per_device_batch_size, drop_last=True) + dataloader = accelerator.prepare(dataloader) + + iterator = iter(dataloader) + next(iterator) # Skip first batch tensor([0, 1, 2, 3, 4], device='cuda:0') + batch = next(iterator) + gathered_items = accelerator.gather_for_metrics(batch) + + # Should return a full set of complete batches from each GPU + num_expected_items = per_device_batch_size * accelerator.num_processes + assert gathered_items.size(0) == (num_expected_items), ( + f"Expected number of items: {num_expected_items}, Actual: {gathered_items.size(0)}" + ) + + +def main(): + dataloader_config = DataLoaderConfiguration(split_batches=False, dispatch_batches=False) + accelerator = Accelerator(dataloader_config=dataloader_config) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + # TorchXLA does not support batch dispatching. 'put_on_device' is always False for + # TorchXLA, which can cause a value error in 'prepare_data_loader' function. + dispatch_batches_options = [False] if accelerator.state.distributed_type == DistributedType.XLA else [True, False] + + # Temporarily close this test for TorchXLA due to the 'Cannot set version_counter for + # inference tensor' error in inference mode. Reopen it after TorchXLA fixes this bug. + # These are a bit slower so they should only be ran on the GPU or TPU + if accelerator.device.type != "cpu" and not is_torch_xla_available(): + if accelerator.is_local_main_process: + print("**Testing gather_for_metrics**") + for split_batches in [True, False]: + for dispatch_batches in dispatch_batches_options: + if accelerator.is_local_main_process: + print(f"With: `split_batches={split_batches}`, `dispatch_batches={dispatch_batches}`") + test_mrpc(dispatch_batches, split_batches) + accelerator.state._reset_state() + print("test_gather_for_metrics_with_iterable_dataset") + test_gather_for_metrics_with_iterable_dataset() + print("test gather_for_metrics_with_non_tensor_objects_iterable_dataset") + test_gather_for_metrics_with_non_tensor_objects_iterable_dataset() + + # MpDeviceLoader in TorchXLA is an asynchronous loader that preloads several batches into cache. + # This can cause the 'end_of_dataloader' of DataLoaderStateMixin to be set earlier than intended. + # Skip this test when TorchXLA is enabled. + if accelerator.state.distributed_type != DistributedType.XLA: + if accelerator.is_local_main_process: + print("**Test torch metrics**") + for split_batches in [True, False]: + for dispatch_batches in dispatch_batches_options: + dataloader_config = DataLoaderConfiguration( + split_batches=split_batches, dispatch_batches=dispatch_batches + ) + accelerator = Accelerator(dataloader_config=dataloader_config) + if accelerator.is_local_main_process: + print(f"With: `split_batches={split_batches}`, `dispatch_batches={dispatch_batches}`, length=99") + test_torch_metrics(accelerator, 99) + accelerator.state._reset_state() + if accelerator.is_local_main_process: + print("**Test last batch is not dropped when perfectly divisible**") + accelerator = Accelerator() + test_torch_metrics(accelerator, 512) + accelerator.state._reset_state() + if accelerator.is_local_main_process: + print("**Test that `drop_last` is taken into account**") + test_gather_for_metrics_drop_last() + accelerator.end_training() + accelerator.state._reset_state() + + +def _mp_fn(index): + # For xla_spawn (TPUs) + main() + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_peak_memory_usage.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_peak_memory_usage.py new file mode 100644 index 0000000000000000000000000000000000000000..ccdf328fe64b5d08ea30407e0210715343fde20c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_peak_memory_usage.py @@ -0,0 +1,323 @@ +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import gc +import json +import os + +import torch +from datasets import load_dataset +from torch.optim import AdamW +from torch.utils.data import DataLoader +from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed + +from accelerate import Accelerator, DistributedType +from accelerate.utils import ( + is_hpu_available, + is_mlu_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_xpu_available, +) +from accelerate.utils.deepspeed import DummyOptim, DummyScheduler + + +MAX_GPU_BATCH_SIZE = 16 +EVAL_BATCH_SIZE = 32 + + +# Converting Bytes to Megabytes +def b2mb(x): + return int(x / 2**20) + + +# This context manager is used to track the peak memory usage of the process +class TorchTracemalloc: + def __enter__(self): + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() # reset the peak gauge to zero + self.begin = torch.cuda.memory_allocated() + elif is_mlu_available(): + torch.mlu.empty_cache() + torch.mlu.reset_max_memory_allocated() # reset the peak gauge to zero + self.begin = torch.mlu.memory_allocated() + elif is_sdaa_available(): + torch.sdaa.empty_cache() + torch.sdaa.reset_max_memory_allocated() # reset the peak gauge to zero + self.begin = torch.sdaa.memory_allocated() + elif is_musa_available(): + torch.musa.empty_cache() + torch.musa.reset_max_memory_allocated() # reset the peak gauge to zero + self.begin = torch.musa.memory_allocated() + elif is_npu_available(): + torch.npu.empty_cache() + torch.npu.reset_max_memory_allocated() # reset the peak gauge to zero + self.begin = torch.npu.memory_allocated() + elif is_xpu_available(): + torch.xpu.empty_cache() + torch.xpu.reset_peak_memory_stats() # reset the peak gauge to zero + self.begin = torch.xpu.memory_allocated() + elif is_hpu_available(): + # torch.hpu.empty_cache() # not available on hpu as it reserves all device memory for the current process + torch.hpu.reset_peak_memory_stats() # reset the peak gauge to zero + self.begin = torch.hpu.memory_allocated() + elif is_neuron_available(): + torch.neuron.empty_cache() + torch.neuron.reset_peak_memory_stats() # reset the peak gauge to zero + self.begin = torch.neuron.memory_allocated() + return self + + def __exit__(self, *exc): + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + self.end = torch.cuda.memory_allocated() + self.peak = torch.cuda.max_memory_allocated() + elif is_mlu_available(): + torch.mlu.empty_cache() + self.end = torch.mlu.memory_allocated() + self.begin = torch.mlu.max_memory_allocated() + elif is_sdaa_available(): + torch.sdaa.empty_cache() + self.end = torch.sdaa.memory_allocated() + self.begin = torch.sdaa.max_memory_allocated() + elif is_musa_available(): + torch.musa.empty_cache() + self.end = torch.musa.memory_allocated() + self.begin = torch.musa.max_memory_allocated() + elif is_npu_available(): + torch.npu.empty_cache() + self.end = torch.npu.memory_allocated() + self.peak = torch.npu.max_memory_allocated() + elif is_xpu_available(): + torch.xpu.empty_cache() + self.end = torch.xpu.memory_allocated() + self.peak = torch.xpu.max_memory_allocated() + elif is_hpu_available(): + # torch.hpu.empty_cache() # not available on hpu as it reserves all device memory for the current process + self.end = torch.hpu.memory_allocated() + self.peak = torch.hpu.max_memory_allocated() + elif is_neuron_available(): + torch.neuron.empty_cache() + self.end = torch.neuron.memory_allocated() + self.peak = torch.neuron.max_memory_allocated() + self.used = b2mb(self.end - self.begin) + self.peaked = b2mb(self.peak - self.begin) + # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}") + + +def get_dataloaders( + accelerator: Accelerator, + batch_size: int = 16, + model_name: str = "bert-base-cased", + n_train: int = 320, + n_val: int = 160, +): + """ + Creates a set of `DataLoader`s for the `glue` dataset. + + Args: + accelerator (`Accelerator`): + An `Accelerator` object + batch_size (`int`, *optional*): + The batch size for the train and validation DataLoaders. + model_name (`str`, *optional*): + The name of the model to use. + n_train (`int`, *optional*): + The number of training examples to use. + n_val (`int`, *optional*): + The number of validation examples to use. + """ + tokenizer = AutoTokenizer.from_pretrained(model_name) + datasets = load_dataset( + "nyu-mll/glue", "mrpc", split={"train": f"train[:{n_train}]", "validation": f"validation[:{n_val}]"} + ) + + def tokenize_function(examples): + # max_length=None => use the model max length (it's actually the default) + outputs = tokenizer(examples["sentence1"], examples["sentence2"], truncation=True, max_length=None) + return outputs + + # Apply the method we just defined to all the examples in all the splits of the dataset + tokenized_datasets = datasets.map( + tokenize_function, batched=True, remove_columns=["idx", "sentence1", "sentence2"], load_from_cache_file=False + ) + + # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the + # transformers library + tokenized_datasets = tokenized_datasets.rename_column("label", "labels") + + def collate_fn(examples): + # On TPU it's best to pad everything to the same length or training will be very slow. + if accelerator.distributed_type == DistributedType.XLA: + return tokenizer.pad(examples, padding="max_length", max_length=128, return_tensors="pt") + return tokenizer.pad(examples, padding="longest", return_tensors="pt") + + # Instantiate dataloaders. + train_dataloader = DataLoader( + tokenized_datasets["train"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size + ) + eval_dataloader = DataLoader( + tokenized_datasets["validation"], shuffle=False, collate_fn=collate_fn, batch_size=EVAL_BATCH_SIZE + ) + + return train_dataloader, eval_dataloader + + +def training_function(config, args): + # Initialize accelerator + accelerator = Accelerator() + + # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs + lr = config["lr"] + num_epochs = int(config["num_epochs"]) + seed = int(config["seed"]) + batch_size = int(config["batch_size"]) + model_name = args.model_name_or_path + + set_seed(seed) + train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size, model_name, args.n_train, args.n_val) + + # Instantiate the model (we build the model here so that the seed also control new weights initialization) + model = AutoModelForSequenceClassification.from_pretrained(model_name, return_dict=True) + + # Instantiate optimizer + optimizer_cls = ( + AdamW + if accelerator.state.deepspeed_plugin is None + or "optimizer" not in accelerator.state.deepspeed_plugin.deepspeed_config + else DummyOptim + ) + optimizer = optimizer_cls(params=model.parameters(), lr=lr) + + if accelerator.state.deepspeed_plugin is not None: + gradient_accumulation_steps = accelerator.state.deepspeed_plugin.deepspeed_config[ + "gradient_accumulation_steps" + ] + else: + gradient_accumulation_steps = 1 + max_training_steps = (len(train_dataloader) * num_epochs) // gradient_accumulation_steps + + # Instantiate scheduler + if ( + accelerator.state.deepspeed_plugin is None + or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config + ): + lr_scheduler = get_linear_schedule_with_warmup( + optimizer=optimizer, + num_warmup_steps=0, + num_training_steps=max_training_steps, + ) + else: + lr_scheduler = DummyScheduler(optimizer, total_num_steps=max_training_steps, warmup_num_steps=0) + + # Prepare everything + # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the + # prepare method. + model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare( + model, optimizer, train_dataloader, eval_dataloader, lr_scheduler + ) + + # We need to keep track of how many total steps we have iterated over + overall_step = 0 + # We also need to keep track of the stating epoch so files are named properly + starting_epoch = 0 + + # Now we train the model + train_total_peak_memory = {} + for epoch in range(starting_epoch, num_epochs): + with TorchTracemalloc() as tracemalloc: + model.train() + for step, batch in enumerate(train_dataloader): + outputs = model(**batch) + loss = outputs.loss + loss = loss / gradient_accumulation_steps + accelerator.backward(loss) + if step % gradient_accumulation_steps == 0: + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + overall_step += 1 + + # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage + accelerator.print(f"Memory before entering the train : {b2mb(tracemalloc.begin)}") + accelerator.print(f"Memory consumed at the end of the train (end-begin): {tracemalloc.used}") + accelerator.print(f"Peak Memory consumed during the train (max-begin): {tracemalloc.peaked}") + accelerator.print( + f"Total Peak Memory consumed during the train (max): {tracemalloc.peaked + b2mb(tracemalloc.begin)}" + ) + train_total_peak_memory[f"epoch-{epoch}"] = tracemalloc.peaked + b2mb(tracemalloc.begin) + if args.peak_memory_upper_bound is not None: + assert train_total_peak_memory[f"epoch-{epoch}"] <= args.peak_memory_upper_bound, ( + "Peak memory usage exceeded the upper bound" + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + with open(os.path.join(args.output_dir, "peak_memory_utilization.json"), "w") as f: + json.dump(train_total_peak_memory, f) + accelerator.end_training() + + +def main(): + parser = argparse.ArgumentParser(description="Simple example of training script tracking peak GPU memory usage.") + parser.add_argument( + "--model_name_or_path", + type=str, + default="bert-base-cased", + help="Path to pretrained model or model identifier from huggingface.co/models.", + required=False, + ) + parser.add_argument( + "--output_dir", + type=str, + default=".", + help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory.", + ) + parser.add_argument( + "--peak_memory_upper_bound", + type=float, + default=None, + help="The upper bound of peak memory usage in MB. If set, the training will throw an error if the peak memory usage exceeds this value.", + ) + parser.add_argument( + "--n_train", + type=int, + default=320, + help="Number of training examples to use.", + ) + parser.add_argument( + "--n_val", + type=int, + default=160, + help="Number of validation examples to use.", + ) + parser.add_argument( + "--num_epochs", + type=int, + default=1, + help="Number of train epochs.", + ) + args = parser.parse_args() + config = {"lr": 2e-5, "num_epochs": args.num_epochs, "seed": 42, "batch_size": 16} + training_function(config, args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_performance.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_performance.py new file mode 100644 index 0000000000000000000000000000000000000000..acdfa5b4216a2b192dc38711550d4cfd24cd1184 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_performance.py @@ -0,0 +1,308 @@ +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import os +from contextlib import nullcontext +from pathlib import Path + +import evaluate +import torch +from datasets import load_dataset +from torch.optim import AdamW +from torch.utils.data import DataLoader +from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup + +from accelerate import Accelerator, DistributedType +from accelerate.parallelism_config import ParallelismConfig +from accelerate.utils import SAFE_WEIGHTS_NAME, set_seed +from accelerate.utils.deepspeed import DummyOptim, DummyScheduler + + +MAX_GPU_BATCH_SIZE = 16 +EVAL_BATCH_SIZE = 32 + + +def get_dataloaders(accelerator: Accelerator, batch_size: int = 16, model_name: str = "bert-base-cased"): + """ + Creates a set of `DataLoader`s for the `glue` dataset. + + Args: + accelerator (`Accelerator`): + An `Accelerator` object + batch_size (`int`, *optional*): + The batch size for the train and validation DataLoaders. + model_name (`str`, *optional*): + """ + tokenizer = AutoTokenizer.from_pretrained(model_name) + + datasets = load_dataset("nyu-mll/glue", "mrpc") + + def tokenize_function(examples): + # max_length=None => use the model max length (it's actually the default) + outputs = tokenizer(examples["sentence1"], examples["sentence2"], truncation=True, max_length=None) + return outputs + + # Apply the method we just defined to all the examples in all the splits of the dataset + tokenized_datasets = datasets.map( + tokenize_function, batched=True, remove_columns=["idx", "sentence1", "sentence2"], load_from_cache_file=False + ) + + # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the + # transformers library + tokenized_datasets = tokenized_datasets.rename_column("label", "labels") + + def collate_fn(examples): + # On TPU it's best to pad everything to the same length or training will be very slow. + if accelerator.distributed_type == DistributedType.XLA: + return tokenizer.pad(examples, padding="max_length", max_length=128, return_tensors="pt") + return tokenizer.pad(examples, padding="longest", return_tensors="pt") + + # Instantiate dataloaders. + train_dataloader = DataLoader( + tokenized_datasets["train"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size + ) + eval_dataloader = DataLoader( + tokenized_datasets["validation"], shuffle=False, collate_fn=collate_fn, batch_size=EVAL_BATCH_SIZE + ) + + return train_dataloader, eval_dataloader + + +def training_function(config, args): + accelerator_kwargs = {} + # need this for DeepSpeed tests as `args.tp_size` would be None and `torch.distributed.init_device_mesh` would fail + if args.tp_size is not None: + accelerator_kwargs["parallelism_config"] = ParallelismConfig(tp_size=args.tp_size) + + # Initialize accelerator + accelerator = Accelerator(**accelerator_kwargs) + + # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs + lr = config["lr"] + num_epochs = int(config["num_epochs"]) + seed = int(config["seed"]) + batch_size = int(config["batch_size"]) + model_name = args.model_name_or_path + + set_seed(seed) + train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size, model_name) + + # Add TP related kwargs if provided + model_kwargs = {} + if args.tp_plan is not None: + model_kwargs["tp_plan"] = args.tp_plan + if args.tp_size is not None: + model_kwargs["tp_size"] = args.tp_size + if args.model_dtype is not None: + model_kwargs["torch_dtype"] = getattr(torch, args.model_dtype) + + # Instantiate the model (we build the model here so that the seed also control new weights initialization) + model = AutoModelForSequenceClassification.from_pretrained(model_name, return_dict=True, **model_kwargs) + + if args.add_pad_token: + if model.config.pad_token_id is None: + model.config.pad_token_id = 0 + + # Instantiate optimizer + optimizer_cls = ( + AdamW + if accelerator.state.deepspeed_plugin is None + or "optimizer" not in accelerator.state.deepspeed_plugin.deepspeed_config + else DummyOptim + ) + optimizer = optimizer_cls(params=model.parameters(), lr=lr) + + max_training_steps = len(train_dataloader) * num_epochs + + # Instantiate scheduler + linear_decay_scheduler = False + if ( + accelerator.state.deepspeed_plugin is None + or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config + ): + lr_scheduler = get_linear_schedule_with_warmup( + optimizer=optimizer, + num_warmup_steps=0, + num_training_steps=max_training_steps, + ) + linear_decay_scheduler = True + else: + lr_scheduler = DummyScheduler(optimizer, total_num_steps=max_training_steps, warmup_num_steps=0) + + # Prepare everything + # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the + # prepare method. + model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare( + model, optimizer, train_dataloader, eval_dataloader, lr_scheduler + ) + + # We also need to keep track of the stating epoch so files are named properly + starting_epoch = 0 + + # Now we train the model + metric = evaluate.load("glue", "mrpc") + best_performance = 0 + performance_metric = {} + expected_lr_after_first_optim_step = lr * ( + 1 - 1 / (max_training_steps / accelerator.num_processes / accelerator.gradient_accumulation_steps) + ) + lr_scheduler_check_completed = False + for epoch in range(starting_epoch, num_epochs): + model.train() + for step, batch in enumerate(train_dataloader): + with accelerator.accumulate(model): + outputs = model(**batch) + loss = outputs.loss + accelerator.backward(loss) + context = nullcontext + if args.tp_plan is not None: + from torch.distributed._tensor.experimental import implicit_replication + + context = implicit_replication + with context(): + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # assert the learning rate after first optimizer step + if ( + accelerator.sync_gradients + and not lr_scheduler_check_completed + and linear_decay_scheduler + and accelerator.state.mixed_precision == "no" + ): + assert lr_scheduler.get_last_lr()[0] == expected_lr_after_first_optim_step, ( + f"Wrong lr found at second step, expected {expected_lr_after_first_optim_step}, got {lr_scheduler.get_last_lr()[0]}" + ) + lr_scheduler_check_completed = True + + model.eval() + samples_seen = 0 + for step, batch in enumerate(eval_dataloader): + # We could avoid this line since we set the accelerator with `device_placement=True`. + batch.to(accelerator.device) + with torch.no_grad(): + outputs = model(**batch) + predictions = outputs.logits.argmax(dim=-1) + # It is slightly faster to call this once, than multiple times + predictions, references = accelerator.gather( + (predictions, batch["labels"]) + ) # If we are in a multiprocess environment, the last batch has duplicates + if accelerator.use_distributed: + if step == len(eval_dataloader) - 1: + predictions = predictions[: len(eval_dataloader.dataset) - samples_seen] + references = references[: len(eval_dataloader.dataset) - samples_seen] + else: + samples_seen += references.shape[0] + metric.add_batch( + predictions=predictions, + references=references, + ) + + eval_metric = metric.compute() + # Use accelerator.print to print only on the main process. + accelerator.print(f"epoch {epoch}:", eval_metric) + performance_metric[f"epoch-{epoch}"] = eval_metric["accuracy"] + + if best_performance < eval_metric["accuracy"]: + best_performance = eval_metric["accuracy"] + + # check that the LR is 0 + if linear_decay_scheduler and accelerator.state.mixed_precision == "no": + assert lr_scheduler.get_last_lr()[0] == 0, ( + f"Wrong lr found at last step, expected 0, got {lr_scheduler.get_last_lr()[0]}" + ) + + if args.performance_lower_bound is not None: + assert args.performance_lower_bound <= best_performance, ( + f"Best performance metric {best_performance} is lower than the lower bound {args.performance_lower_bound}" + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + json.dump(performance_metric, f) + + # TODO: skip saving of the model test for TP until the feature lands + if args.tp_plan is None: + # Finally try saving the model + accelerator.save_model(model, args.output_dir) + accelerator.wait_for_everyone() + if args.tp_plan is None: + assert Path(args.output_dir, SAFE_WEIGHTS_NAME).exists(), ( + "Model was not saved when calling `Accelerator.save_model`" + ) + accelerator.end_training() + + +def main(): + parser = argparse.ArgumentParser(description="Simple example of training script tracking peak GPU memory usage.") + parser.add_argument( + "--model_name_or_path", + type=str, + default="bert-base-cased", + help="Path to pretrained model or model identifier from huggingface.co/models.", + required=False, + ) + parser.add_argument( + "--output_dir", + type=str, + default=".", + help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory.", + ) + parser.add_argument( + "--performance_lower_bound", + type=float, + default=None, + help="Optional lower bound for the performance metric. If set, the training will throw error when the performance metric drops below this value.", + ) + parser.add_argument( + "--num_epochs", + type=int, + default=3, + help="Number of train epochs.", + ) + parser.add_argument( + "--add_pad_token", + type=bool, + default=False, + help="To add pad token if not exists.", + ) + parser.add_argument( + "--tp_plan", + type=str, + default=None, + help="pass 'auto' to use TP", + ) + parser.add_argument( + "--tp_size", + type=int, + default=None, + help="TP size to be used to shard the model", + ) + parser.add_argument( + "--model_dtype", + type=str, + default=None, + choices=["bfloat16", "float16", "float32"], + help="Optional dtype to load the model in (defaults to the checkpoint dtype).", + ) + args = parser.parse_args() + config = {"lr": 2e-5, "num_epochs": args.num_epochs, "seed": 42, "batch_size": 16} + training_function(config, args) + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_pippy.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_pippy.py new file mode 100644 index 0000000000000000000000000000000000000000..1dbd86c46b4a0c12df8ea4d736c7cd1e03f81813 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_pippy.py @@ -0,0 +1,117 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +from transformers import ( + BertConfig, + BertForMaskedLM, + GPT2Config, + GPT2ForSequenceClassification, +) + +from accelerate import PartialState +from accelerate.inference import prepare_pippy +from accelerate.test_utils import torch_device +from accelerate.utils import DistributedType, set_seed + + +model_to_config = { + "bert": (BertForMaskedLM, BertConfig, 512), + "gpt2": (GPT2ForSequenceClassification, GPT2Config, 1024), +} + + +def get_model_and_data_for_text(model_name, device, num_processes: int = 2): + initializer, config, seq_len = model_to_config[model_name] + config_args = {} + # Eventually needed for batch inference tests on gpt-2 when bs != 1 + # if model_name == "gpt2": + # config_args["pad_token_id"] = 0 + model_config = config(**config_args) + model = initializer(model_config) + kwargs = dict(low=0, high=model_config.vocab_size, device=device, dtype=torch.int64, requires_grad=False) + trace_input = torch.randint(size=(1, seq_len), **kwargs) + inference_inputs = torch.randint(size=(num_processes, seq_len), **kwargs) + return model, trace_input, inference_inputs + + +def test_bert(batch_size: int = 2): + set_seed(42) + state = PartialState() + model, trace_input, inference_inputs = get_model_and_data_for_text("bert", "cpu", batch_size) + model = prepare_pippy(model, example_args=(trace_input,), no_split_module_classes=model._no_split_modules) + # For inference args need to be a tuple + inputs = inference_inputs.to(torch_device) + with torch.no_grad(): + output = model(inputs) + # Zach: Check that we just grab the real outputs we need at the end + if not state.is_last_process: + assert output is None, "Output was not generated on just the last process!" + else: + assert output is not None, "Output was not generated in the last process!" + + +def test_gpt2(batch_size: int = 2): + set_seed(42) + state = PartialState() + model, trace_input, inference_inputs = get_model_and_data_for_text("gpt2", "cpu", batch_size) + model = prepare_pippy(model, example_args=(trace_input,), no_split_module_classes=model._no_split_modules) + # For inference args need to be a tuple + inputs = inference_inputs.to(torch_device) + with torch.no_grad(): + output = model(inputs) + # Zach: Check that we just grab the real outputs we need at the end + if not state.is_last_process: + assert output is None, "Output was not generated on just the last process!" + else: + assert output is not None, "Output was not generated in the last process!" + + +# Currently disabled, enable again once PyTorch pippy interface can trace a resnet34 +# def test_resnet(batch_size: int = 2): +# set_seed(42) +# state = PartialState() +# model = resnet34() +# input_tensor = torch.rand(1, 3, 224, 224) +# model = prepare_pippy( +# model, +# example_args=(input_tensor,), +# ) +# inference_inputs = torch.rand(batch_size, 3, 224, 224) +# inputs = send_to_device(inference_inputs, torch_device) +# with torch.no_grad(): +# output = model(inputs) +# # Zach: Check that we just grab the real outputs we need at the end +# if not state.is_last_process: +# assert output is None, "Output was not generated on just the last process!" +# else: +# assert output is not None, "Output was not generated in the last process!" + + +if __name__ == "__main__": + state = PartialState() + state.print("Testing pippy integration...") + try: + if state.distributed_type in [DistributedType.MULTI_GPU, DistributedType.MULTI_XPU, DistributedType.MULTI_HPU]: + state.print("Testing GPT2...") + test_gpt2() + # Issue: When modifying the tokenizer for batch GPT2 inference, there's an issue + # due to references + # NameError: cannot access free variable 'chunk_args_list' where it is not associated with a value in enclosing scope + # test_gpt2(3) + state.print("Testing BERT...") + test_bert() + else: + print("Less than two GPUs found, not running tests!") + finally: + state.destroy_process_group() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_zero3_integration.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_zero3_integration.py new file mode 100644 index 0000000000000000000000000000000000000000..f6e46d342a0c754cf1ed5b2629290d62e7c943aa --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/external_deps/test_zero3_integration.py @@ -0,0 +1,59 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch.distributed + +from accelerate.test_utils import require_huggingface_suite, torch_device +from accelerate.utils import is_transformers_available + + +if is_transformers_available(): + from transformers import AutoModel, TrainingArguments + + +GPT2_TINY = "sshleifer/tiny-gpt2" + + +@require_huggingface_suite +def init_torch_dist_then_launch_deepspeed(): + if torch_device == "xpu": + backend = "xccl" + elif torch_device == "hpu": + backend = "hccl" + else: + backend = "nccl" + + torch.distributed.init_process_group(backend=backend) + deepspeed_config = { + "zero_optimization": { + "stage": 3, + }, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + } + train_args = TrainingArguments( + output_dir="./", + deepspeed=deepspeed_config, + ) + model = AutoModel.from_pretrained(GPT2_TINY) + assert train_args is not None + assert model is not None + + +def main(): + init_torch_dist_then_launch_deepspeed() + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_cli.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..fc9dd1d36e8f2949d6e3cebb8ce65efbf0b9e5e4 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_cli.py @@ -0,0 +1,32 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from accelerate.utils import is_xpu_available + + +def main(): + accelerator_type = "GPU" + num_accelerators = 0 + if torch.cuda.is_available(): + num_accelerators = torch.cuda.device_count() + accelerator_type = "GPU" + elif is_xpu_available(): + num_accelerators = torch.xpu.device_count() + accelerator_type = "XPU" + print(f"Successfully ran on {num_accelerators} {accelerator_type}s") + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_ddp_comm_hook.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_ddp_comm_hook.py new file mode 100644 index 0000000000000000000000000000000000000000..0db5844e026d1c035670e518a8f81d33136ea665 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_ddp_comm_hook.py @@ -0,0 +1,85 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from accelerate import Accelerator, DDPCommunicationHookType, DistributedDataParallelKwargs, PartialState +from accelerate.utils import is_hpu_available + + +class MockModel(torch.nn.Module): + def __init__(self): + super().__init__() + torch.manual_seed(0) + self.p = torch.nn.Parameter(torch.randn(40, 20)) + + def forward(self, x, rank): + return self.p * (x ** (1 + rank)) + + +def _run_and_get_grads(model, rank): + torch.manual_seed(2024) + input = torch.randn(40, 20) + output = model(input, rank) + output.mean().backward() + param = next(model.parameters()) + return param.grad + + +def test_ddp_comm_hook(comm_hook, comm_wrapper, comm_state_option): + ddp_kwargs = DistributedDataParallelKwargs( + comm_hook=comm_hook, + comm_wrapper=comm_wrapper, + comm_state_option=comm_state_option, + ) + accelerator = Accelerator(kwargs_handlers=[ddp_kwargs]) + + model = accelerator.prepare(MockModel()) + hook_grads = _run_and_get_grads(model, accelerator.local_process_index) + + reference_model = torch.nn.parallel.DistributedDataParallel( + MockModel().to(accelerator.device), + device_ids=[accelerator.local_process_index], + output_device=accelerator.local_process_index, + ) + reference_grads = _run_and_get_grads(reference_model, accelerator.local_process_index) + + torch.testing.assert_close(hook_grads, reference_grads, rtol=1e-2, atol=1e-2) + + +def main(): + for comm_hook, comm_wrapper, comm_state_option in [ + (DDPCommunicationHookType.NO, DDPCommunicationHookType.NO, {}), + (DDPCommunicationHookType.FP16, DDPCommunicationHookType.NO, {}), + (DDPCommunicationHookType.BF16, DDPCommunicationHookType.NO, {}), + (DDPCommunicationHookType.POWER_SGD, DDPCommunicationHookType.NO, {}), + (DDPCommunicationHookType.POWER_SGD, DDPCommunicationHookType.FP16, {}), + (DDPCommunicationHookType.POWER_SGD, DDPCommunicationHookType.BF16, {}), + (DDPCommunicationHookType.POWER_SGD, DDPCommunicationHookType.NO, {"matrix_approximation_rank": 2}), + (DDPCommunicationHookType.BATCHED_POWER_SGD, DDPCommunicationHookType.NO, {}), + (DDPCommunicationHookType.BATCHED_POWER_SGD, DDPCommunicationHookType.FP16, {}), + (DDPCommunicationHookType.BATCHED_POWER_SGD, DDPCommunicationHookType.BF16, {}), + ]: + if is_hpu_available(): + HPU_UNSUPPORTED_COMM_HOOKS = {DDPCommunicationHookType.FP16, DDPCommunicationHookType.BF16} + if comm_hook in HPU_UNSUPPORTED_COMM_HOOKS or comm_wrapper in HPU_UNSUPPORTED_COMM_HOOKS: + print(f"Skipping test DDP comm hook: {comm_hook}, comm wrapper: {comm_wrapper} on HPU") + continue + + print(f"Test DDP comm hook: {comm_hook}, comm wrapper: {comm_wrapper}") + test_ddp_comm_hook(comm_hook, comm_wrapper, comm_state_option) + PartialState().destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_distributed_data_loop.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_distributed_data_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..84a9247a6bce04a55a67ad33e1761c627772aced --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_distributed_data_loop.py @@ -0,0 +1,429 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pickle +import tempfile +import warnings +from unittest.mock import Mock + +import torch +from torch.utils.data import ( + BatchSampler, + DataLoader, + Dataset, + IterableDataset, + RandomSampler, + TensorDataset, + default_collate, +) + +from accelerate.accelerator import Accelerator, DataLoaderConfiguration +from accelerate.utils.dataclasses import DistributedType + + +NUM_ELEMENTS = 22 +NUM_WORKERS = 4 +BATCH_SIZE = 4 + + +class DummyDataset(Dataset): + def __len__(self): + return NUM_ELEMENTS + + def __getitem__(self, index): + squeeze = False + + if isinstance(index, int): + index = [index] + squeeze = True + elif isinstance(index, slice): + index = list(range(*index.indices(self.size))) + else: + index = list(index) + + batch = [{"index": i, "label": i % 2, "random_augmentation": torch.rand(1).item()} for i in index] + + if squeeze: + batch = batch[0] + + return batch + + +class DummyIterableDataset(IterableDataset): + def __init__(self, data): + self.data = data + + def __iter__(self): + yield from self.data + + +def create_accelerator(even_batches=True): + dataloader_config = DataLoaderConfiguration(even_batches=even_batches) + accelerator = Accelerator(dataloader_config=dataloader_config) + assert accelerator.num_processes == 2, "this script expects that two GPUs are available" + return accelerator + + +def create_dataloader( + accelerator: Accelerator, dataset_size: int, batch_size: int, iterable: bool = False, shuffle: bool = False +): + """ + Create a simple DataLoader to use during the test cases + """ + values = torch.as_tensor(range(dataset_size)) + if shuffle: + values = values[torch.randperm(values.size(0))] + if iterable: + dataset = DummyIterableDataset(values) + else: + dataset = TensorDataset(torch.as_tensor(range(dataset_size))) + + dl = DataLoader(dataset, batch_size=batch_size) + dl = accelerator.prepare(dl) + + return dl + + +def verify_dataloader_batch_sizes( + accelerator: Accelerator, + dataset_size: int, + batch_size: int, + process_0_expected_batch_sizes: list[int], + process_1_expected_batch_sizes: list[int], +): + """ + A helper function for verifying the batch sizes coming from a prepared dataloader in each process + """ + dl = create_dataloader(accelerator=accelerator, dataset_size=dataset_size, batch_size=batch_size) + + batch_sizes = [len(batch[0]) for batch in dl] + + if accelerator.process_index == 0: + assert batch_sizes == process_0_expected_batch_sizes + elif accelerator.process_index == 1: + assert batch_sizes == process_1_expected_batch_sizes + + +def test_default_ensures_even_batch_sizes(): + accelerator = create_accelerator() + + # without padding, we would expect a different number of batches + verify_dataloader_batch_sizes( + accelerator, + dataset_size=3, + batch_size=1, + process_0_expected_batch_sizes=[1, 1], + process_1_expected_batch_sizes=[1, 1], + ) + + # without padding, we would expect the same number of batches, but different sizes + verify_dataloader_batch_sizes( + accelerator, + dataset_size=7, + batch_size=2, + process_0_expected_batch_sizes=[2, 2], + process_1_expected_batch_sizes=[2, 2], + ) + + +def test_can_disable_even_batches(): + accelerator = create_accelerator(even_batches=False) + + verify_dataloader_batch_sizes( + accelerator, + dataset_size=3, + batch_size=1, + process_0_expected_batch_sizes=[1, 1], + process_1_expected_batch_sizes=[1], + ) + + verify_dataloader_batch_sizes( + accelerator, + dataset_size=7, + batch_size=2, + process_0_expected_batch_sizes=[2, 2], + process_1_expected_batch_sizes=[2, 1], + ) + + +def test_can_join_uneven_inputs(): + accelerator = create_accelerator(even_batches=False) + + model = torch.nn.Linear(1, 1) + ddp_model = accelerator.prepare(model) + + dl = create_dataloader(accelerator, dataset_size=3, batch_size=1) + + batch_idxs = [] + with accelerator.join_uneven_inputs([ddp_model]): + for batch_idx, batch in enumerate(dl): + output = ddp_model(batch[0].float()) + loss = output.sum() + loss.backward() + batch_idxs.append(batch_idx) + + accelerator.wait_for_everyone() + + if accelerator.process_index == 0: + assert batch_idxs == [0, 1] + elif accelerator.process_index == 1: + assert batch_idxs == [0] + + +def test_join_raises_warning_for_non_ddp_distributed(accelerator): + with warnings.catch_warnings(record=True) as w: + with accelerator.join_uneven_inputs([Mock()]): + pass + + assert issubclass(w[-1].category, UserWarning) + assert "only supported for multi-GPU" in str(w[-1].message) + + +def test_join_can_override_even_batches(): + default_even_batches = True + overridden_even_batches = False + accelerator = create_accelerator(even_batches=default_even_batches) + model = torch.nn.Linear(1, 1) + ddp_model = accelerator.prepare(model) + train_dl = create_dataloader(accelerator, dataset_size=3, batch_size=1) + valid_dl = create_dataloader(accelerator, dataset_size=3, batch_size=1) + + with accelerator.join_uneven_inputs([ddp_model], even_batches=overridden_even_batches): + train_dl_overridden_value = train_dl.batch_sampler.even_batches + valid_dl_overridden_value = valid_dl.batch_sampler.even_batches + + assert train_dl_overridden_value == overridden_even_batches + assert valid_dl_overridden_value == overridden_even_batches + assert train_dl.batch_sampler.even_batches == default_even_batches + assert valid_dl.batch_sampler.even_batches == default_even_batches + + +def test_join_can_override_for_mixed_type_dataloaders(): + default_even_batches = True + overridden_even_batches = False + accelerator = create_accelerator(even_batches=default_even_batches) + model = torch.nn.Linear(1, 1) + ddp_model = accelerator.prepare(model) + create_dataloader(accelerator, dataset_size=3, batch_size=1, iterable=True) + batch_dl = create_dataloader(accelerator, dataset_size=3, batch_size=1) + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + try: + with accelerator.join_uneven_inputs([ddp_model], even_batches=overridden_even_batches): + batch_dl_overridden_value = batch_dl.batch_sampler.even_batches + except AttributeError: + # ensure attribute error is not raised when processing iterable dl + raise AssertionError + + assert batch_dl_overridden_value == overridden_even_batches + assert batch_dl.batch_sampler.even_batches == default_even_batches + + +def test_join_raises_warning_for_iterable_when_overriding_even_batches(): + accelerator = create_accelerator() + model = torch.nn.Linear(1, 1) + ddp_model = accelerator.prepare(model) + create_dataloader(accelerator, dataset_size=3, batch_size=1, iterable=True) + + with warnings.catch_warnings(record=True) as w: + with accelerator.join_uneven_inputs([ddp_model], even_batches=False): + pass + + assert issubclass(w[-1].category, UserWarning) + assert "only supported for map-style datasets" in str(w[-1].message) + + +def test_pickle_accelerator(): + accelerator = create_accelerator() + data_loader = create_dataloader(accelerator, dataset_size=32, batch_size=4) + _ = accelerator.prepare(data_loader) + pickled_accelerator = pickle.dumps(accelerator) + unpickled_accelerator = pickle.loads(pickled_accelerator) + # TODO: Maybe this should be implemented as __eq__ for AcceleratorState? + assert accelerator.state.__dict__ == unpickled_accelerator.state.__dict__ + + +def test_data_loader(data_loader, accelerator): + # Prepare the DataLoader + data_loader = accelerator.prepare(data_loader) + + all_examples = [] + for i, batch in enumerate(data_loader): + index, _ = accelerator.gather_for_metrics((batch["index"], batch["label"])) + all_examples.extend(index.detach().cpu().numpy().tolist()) + + # Sort the examples + sorted_all_examples = sorted(all_examples) + + # Check if all elements are present in the sorted list of iterated samples + assert len(set(sorted_all_examples)) == NUM_ELEMENTS, ( + "Not all the dataset elements have been iterated in an epoch due to duplication of samples across processes." + ) + + +def _test_stateful_dataloader_resume(accelerator, iterable): + """ + Helper: iterate a stateful dataloader, save state after a few batches using `load_state_dict`, + resume from the saved state, and verify the resumed batches match what was originally unseen. + + Saves early (after 3 batches) so many batches remain, exposing any off-by-one in state restoration. + Tested with both iterable and map-style datasets to cover different state_dict code paths. + """ + old_dataloader_config = accelerator.dataloader_config + try: + accelerator.dataloader_config = DataLoaderConfiguration(use_stateful_dataloader=True) + prepared_dl = create_dataloader( + accelerator, dataset_size=32 * accelerator.num_processes, batch_size=4, iterable=iterable, shuffle=True + ) + untrained_batches = [] + save_step = 2 + for step, batch in enumerate(prepared_dl): + if step == save_step: + state_dict = prepared_dl.state_dict() + if step > save_step: + untrained_batches.append(batch) + not_skipped_batches = accelerator.gather(untrained_batches) + prepared_dl.load_state_dict(state_dict) + resumed_batches = [] + for batch in prepared_dl: + resumed_batches.append(batch) + resumed_batches = accelerator.gather(resumed_batches) + assert len(not_skipped_batches) == len(resumed_batches), ( + f"Expected {len(not_skipped_batches)} batches after resume, got {len(resumed_batches)}" + ) + for b1, b2 in zip(not_skipped_batches, resumed_batches): + for v1, v2 in zip(b1, b2): + assert torch.equal(v1, v2), f"Batch {b1} and {b2} are not equal" + finally: + accelerator.dataloader_config = old_dataloader_config + + +def test_stateful_dataloader(accelerator): + """ + Tests that a stateful dataloader can be iterated over, saved after a few batches using `load_state_dict`, and then + resumed from the saved state. + + The result should be the same as the rest of the data that iterated over after saving. + """ + _test_stateful_dataloader_resume(accelerator, iterable=True) + _test_stateful_dataloader_resume(accelerator, iterable=False) + + +def _test_stateful_dataloader_save_state_resume(accelerator, iterable): + """ + Helper: iterate a stateful dataloader, save state after a few batches using `Accelerator.save_state`, + resume, and verify the resumed batches match what was originally unseen. + """ + old_dataloader_config = accelerator.dataloader_config + try: + with tempfile.TemporaryDirectory() as tmpdir: + accelerator.dataloader_config = DataLoaderConfiguration(use_stateful_dataloader=True) + prepared_dl = create_dataloader( + accelerator, dataset_size=32 * accelerator.num_processes, batch_size=4, iterable=iterable, shuffle=True + ) + untrained_batches = [] + save_step = 2 + for step, batch in enumerate(prepared_dl): + if step == save_step: + accelerator.save_state(tmpdir) + if step > save_step: + untrained_batches.append(batch) + not_skipped_batches = accelerator.gather(untrained_batches) + accelerator.load_state(tmpdir) + resumed_batches = [] + for batch in prepared_dl: + resumed_batches.append(batch) + resumed_batches = accelerator.gather(resumed_batches) + assert len(not_skipped_batches) == len(resumed_batches), ( + f"Expected {len(not_skipped_batches)} batches after resume, got {len(resumed_batches)}" + ) + for b1, b2 in zip(not_skipped_batches, resumed_batches): + for v1, v2 in zip(b1, b2): + assert torch.equal(v1, v2), f"Batch {b1} and {b2} are not equal" + finally: + accelerator.dataloader_config = old_dataloader_config + + +def test_stateful_dataloader_save_state(accelerator): + """ + Tests that a stateful dataloader can be iterated over, saved after a few batches using `Accelerator.save_state`, + and then resumed from the saved state. + + The result should be the same as the rest of the data that iterated over after saving. + """ + _test_stateful_dataloader_save_state_resume(accelerator, iterable=True) + _test_stateful_dataloader_save_state_resume(accelerator, iterable=False) + + +def main(): + accelerator = create_accelerator() + torch.manual_seed(accelerator.process_index) + + accelerator.print("Test that even_batches variable ensures uniform batches across processes") + test_default_ensures_even_batch_sizes() + + accelerator.print("Run tests with even_batches disabled") + test_can_disable_even_batches() + + accelerator.print("Test joining uneven inputs") + test_can_join_uneven_inputs() + + accelerator.print("Test overriding even_batches when joining uneven inputs") + test_join_can_override_even_batches() + + accelerator.print("Test overriding even_batches for mixed dataloader types") + test_join_can_override_for_mixed_type_dataloaders() + + accelerator.print("Test overriding even_batches raises a warning for iterable dataloaders") + test_join_raises_warning_for_iterable_when_overriding_even_batches() + + accelerator.print("Test join with non DDP distributed raises warning") + original_state = accelerator.state.distributed_type + accelerator.state.distributed_type = DistributedType.FSDP + test_join_raises_warning_for_non_ddp_distributed(accelerator) + accelerator.state.distributed_type = original_state + + accelerator.print("Test pickling an accelerator") + test_pickle_accelerator() + + dataset = DummyDataset() + + accelerator.print("Test DataLoader with shuffle=False") + loader = DataLoader(dataset, shuffle=False, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS) + test_data_loader(loader, accelerator) + + accelerator.print("Test DataLoader with shuffle=True") + loader = DataLoader(dataset, shuffle=True, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS) + test_data_loader(loader, accelerator) + + accelerator.print("Test DataLoader with batch_sampler") + sampler = BatchSampler(RandomSampler(dataset), batch_size=BATCH_SIZE, drop_last=False) + loader = DataLoader(dataset, batch_sampler=sampler, num_workers=NUM_WORKERS) + test_data_loader(loader, accelerator) + + accelerator.print("Test DataLoader with sampler as an instance of `BatchSampler`") + sampler = BatchSampler(RandomSampler(dataset), batch_size=BATCH_SIZE, drop_last=False) + loader = DataLoader(dataset, sampler=sampler, batch_size=None, collate_fn=default_collate, num_workers=NUM_WORKERS) + test_data_loader(loader, accelerator) + test_stateful_dataloader(accelerator) + test_stateful_dataloader_save_state(accelerator) + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_merge_weights.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_merge_weights.py new file mode 100644 index 0000000000000000000000000000000000000000..f280c8fa17919367001389a8efcd78faaa041f0c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_merge_weights.py @@ -0,0 +1,158 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import logging +import shutil +from pathlib import Path + +import torch +from safetensors.torch import load_file +from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy, StateDictType +from torch.utils.data import DataLoader + +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.commands.merge import merge_command, merge_command_parser +from accelerate.state import AcceleratorState +from accelerate.test_utils import torch_device +from accelerate.test_utils.training import RegressionDataset +from accelerate.utils import merge_fsdp_weights, patch_environment, save_fsdp_model + + +logging.basicConfig(level=logging.INFO) + +parser = merge_command_parser() + + +class TinyModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.linear1 = torch.nn.Linear(16, 16) + self.activation = torch.nn.ReLU() + self.linear2 = torch.nn.Linear(16, 16) + self.softmax = torch.nn.Softmax() + + def forward(self, x): + return self.linear2(self.activation(self.linear1(x))) + + +def setup(): + if AcceleratorState._shared_state != {}: + AcceleratorState()._reset_state() + plugin = FullyShardedDataParallelPlugin( + sharding_strategy=ShardingStrategy.FULL_SHARD, state_dict_type=StateDictType.SHARDED_STATE_DICT + ) + model = TinyModel() + with patch_environment(fsdp_auto_wrap_policy="SIZE_BASED_WRAP"): + plugin.set_auto_wrap_policy(model) + accelerator = Accelerator(fsdp_plugin=plugin) + model = accelerator.prepare(model) + return model, plugin, accelerator + + +def mock_training(accelerator, model): + train_set = RegressionDataset(length=128, seed=42) + train_dl = DataLoader(train_set, batch_size=16, shuffle=False) + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + + train_dl, model, optimizer = accelerator.prepare(train_dl, model, optimizer) + for _ in range(3): + for batch in train_dl: + model.zero_grad() + output = model(batch["x"]) + loss = torch.nn.functional.mse_loss(output, batch["y"]) + accelerator.backward(loss) + optimizer.step() + return model + + +def check_weights(operation, state_1, state_2): + for weight_1, weight_2 in zip(state_1.values(), state_2.values()): + if operation == "same": + assert torch.allclose(weight_1, weight_2) + else: + assert not torch.allclose(weight_1, weight_2) + + +def check_safetensors_weights(path, model): + safe_state_dict = load_file(path / "model.safetensors") + safe_loaded_model = TinyModel().to(torch_device) + check_weights("diff", model.state_dict(), safe_loaded_model.state_dict()) + safe_loaded_model.load_state_dict(safe_state_dict) + check_weights("same", model.state_dict(), safe_loaded_model.state_dict()) + + +def check_pytorch_weights(path, model): + nonsafe_state_dict = torch.load(path / "pytorch_model.bin", weights_only=True) + nonsafe_loaded_model = TinyModel().to(torch_device) + check_weights("diff", model.state_dict(), nonsafe_loaded_model.state_dict()) + nonsafe_loaded_model.load_state_dict(nonsafe_state_dict) + check_weights("same", model.state_dict(), nonsafe_loaded_model.state_dict()) + + +def test_merge_weights_safetensors(model, path): + # Should now be saved at `path/merged.safetensors` + merge_fsdp_weights(path / "pytorch_model_fsdp_0", path, safe_serialization=True) + check_safetensors_weights(path, model) + + +def test_merge_weights_command_safetensors(model, path): + args = parser.parse_args([str(path / "pytorch_model_fsdp_0"), str(path)]) + merge_command(args) + check_safetensors_weights(path, model) + + +def test_merge_weights_pytorch(model, path): + # Should now be saved at `path/merged.bin` + merge_fsdp_weights(path / "pytorch_model_fsdp_0", path, safe_serialization=False) + check_pytorch_weights(path, model) + + +def test_merge_weights_command_pytorch(model, path): + args = parser.parse_args([str(path / "pytorch_model_fsdp_0"), str(path), "--unsafe_serialization"]) + merge_command(args) + check_pytorch_weights(path, model) + + +if __name__ == "__main__": + # Note this test requires at least two accelerators! + model, plugin, accelerator = setup() + if accelerator.num_processes > 1: + try: + # Initial setup for things + out_path = Path("test_merge_weights_fsdp_weights") + if not out_path.exists(): + out_path.mkdir(parents=True, exist_ok=True) + + # Train briefly once weights aren't the baseline + model = mock_training(accelerator, model) + accelerator.wait_for_everyone() + + gc.collect() # Needed for some lingering refs after training + save_fsdp_model(plugin, accelerator, model, out_path) + accelerator.wait_for_everyone() + + # Finally we can test + test_merge_weights_safetensors(model, out_path) + test_merge_weights_command_safetensors(model, out_path) + test_merge_weights_pytorch(model, out_path) + test_merge_weights_command_pytorch(model, out_path) + except Exception: + raise + finally: + # Cleanup in case of any failures + if accelerator.is_main_process: + shutil.rmtree(out_path) + accelerator.wait_for_everyone() + accelerator.end_training() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_notebook.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_notebook.py new file mode 100644 index 0000000000000000000000000000000000000000..bcfd51b26b2fb4b65d7f3101ccbfbda9fc9e703c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_notebook.py @@ -0,0 +1,150 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test file to ensure that in general certain situational setups for notebooks work. +""" + +import os +import time + +from pytest import mark, raises +from torch.distributed.elastic.multiprocessing.errors import ChildFailedError + +from accelerate import PartialState, notebook_launcher +from accelerate.test_utils import require_bnb +from accelerate.utils import is_bnb_available, is_rocm_available, is_xpu_available + + +def basic_function(): + # Just prints the PartialState + print(f"PartialState:\n{PartialState()}") + + +def tough_nut_function(queue): + if queue.empty(): + return + trial = queue.get() + if trial > 0: + queue.put(trial - 1) + raise RuntimeError("The nut hasn't cracked yet! Try again.") + + print(f"PartialState:\n{PartialState()}") + + +def bipolar_sleep_function(sleep_sec: int): + state = PartialState() + if state.process_index % 2 == 0: + raise RuntimeError("I'm an even process. I don't like to sleep.") + else: + time.sleep(sleep_sec) + + +def assert_clean_cuda_fork_function(): + # Regression guard: notebook_launcher must not load the CUDA driver in the + # parent before forking children. If it does, _is_in_bad_fork is set in each + # forked child and any later CUDA call (e.g. Accelerator() -> set_device) + # fails with "Cannot re-initialize CUDA in forked subprocess". + import torch + + if torch.cuda.is_available(): + assert not torch._C._cuda_isInBadFork(), ( + "Child process started in bad-fork state — notebook_launcher tainted " + "the parent's CUDA driver before forking." + ) + + +NUM_PROCESSES = int(os.environ.get("ACCELERATE_NUM_PROCESSES", 1)) +require_multi_process = mark.skipif(NUM_PROCESSES < 2, reason="Need at least 2 processes") + + +def test_can_initialize(): + notebook_launcher(basic_function, (), num_processes=NUM_PROCESSES) + + +@require_multi_process +def test_clean_cuda_fork(): + # Regression test for the CUDA-driver-loaded-in-parent bug that broke + # notebook_launcher on accelerate >=1.9 and torch >=2.10 (see issue #3925). + notebook_launcher(assert_clean_cuda_fork_function, (), num_processes=NUM_PROCESSES) + + +@require_multi_process +def test_static_rdzv_backend(): + notebook_launcher(basic_function, (), num_processes=NUM_PROCESSES, rdzv_backend="static") + + +@require_multi_process +def test_c10d_rdzv_backend(): + notebook_launcher(basic_function, (), num_processes=NUM_PROCESSES, rdzv_backend="c10d") + + +@require_multi_process +def test_fault_tolerant(max_restarts: int = 3): + # Use torch.multiprocessing to get the right context for the current device + import torch.multiprocessing as mp + + # Get appropriate context - 'spawn' for XPU/ROCm (matches notebook_launcher's start_method), + # 'fork' for others + if is_xpu_available() or is_rocm_available(): + ctx = mp.get_context("spawn") + else: + ctx = mp.get_context("fork") + queue = ctx.Queue() + queue.put(max_restarts) + notebook_launcher(tough_nut_function, (queue,), num_processes=NUM_PROCESSES, max_restarts=max_restarts) + + +@require_multi_process +def test_monitoring(monitor_interval: float = 0.01, sleep_sec: int = 100): + start_time = time.time() + with raises(ChildFailedError, match="I'm an even process. I don't like to sleep."): + notebook_launcher( + bipolar_sleep_function, + (sleep_sec,), + num_processes=NUM_PROCESSES, + monitor_interval=monitor_interval, + ) + assert time.time() - start_time < sleep_sec, "Monitoring did not stop the process in time." + + +@require_bnb +def test_problematic_imports(): + with raises(RuntimeError, match="Please keep these imports"): + import bitsandbytes as bnb # noqa: F401 + + notebook_launcher(basic_function, (), num_processes=NUM_PROCESSES) + + +def main(): + print("Test basic notebook can be ran") + test_can_initialize() + print("Test clean CUDA fork (regression test for #3925)") + test_clean_cuda_fork() + print("Test static rendezvous backend") + test_static_rdzv_backend() + print("Test c10d rendezvous backend") + test_c10d_rdzv_backend() + print("Test fault tolerant") + test_fault_tolerant() + print("Test monitoring") + test_monitoring() + if is_bnb_available(): + print("Test problematic imports (bnb)") + test_problematic_imports() + if NUM_PROCESSES > 1: + PartialState().destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_ops.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2c3f8e5940f80a6dfbb59d3718e23bbc724ced6e --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_ops.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python + +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from accelerate import PartialState +from accelerate.test_utils.testing import assert_exception +from accelerate.utils.dataclasses import DistributedType +from accelerate.utils.operations import ( + DistributedOperationException, + broadcast, + copy_tensor_to_devices, + gather, + gather_object, + pad_across_processes, + reduce, +) + + +def create_tensor(state): + return (torch.arange(state.num_processes) + 1.0 + (state.num_processes * state.process_index)).to(state.device) + + +def test_gather(state): + tensor = create_tensor(state) + gathered_tensor = gather(tensor) + assert gathered_tensor.tolist() == list(range(1, state.num_processes**2 + 1)) + + +def test_gather_object(state): + # Gather objects in TorchXLA is not supported. + if state.distributed_type == DistributedType.XLA: + return + obj = [state.process_index] + gathered_obj = gather_object(obj) + assert len(gathered_obj) == state.num_processes, f"{gathered_obj}, {len(gathered_obj)} != {state.num_processes}" + assert gathered_obj == list(range(state.num_processes)), f"{gathered_obj} != {list(range(state.num_processes))}" + + +def test_gather_non_contiguous(state): + # Skip this test because the 'is_contiguous' function of XLA tensor always returns True. + if state.distributed_type == DistributedType.XLA: + return + + # Create a non-contiguous tensor (enforce non-contiguity after device memory allocation) + tensor = torch.arange(12, device=state.device).view(4, 3).t() + assert not tensor.is_contiguous() + # Shouldn't error out + _ = gather(tensor) + + +def test_broadcast(state): + tensor = create_tensor(state) + broadcasted_tensor = broadcast(tensor) + assert broadcasted_tensor.shape == torch.Size([state.num_processes]) + assert broadcasted_tensor.tolist() == list(range(1, state.num_processes + 1)) + + +def test_pad_across_processes(state): + # We need to pad the tensor with one more element if we are the main process + # to ensure that we can pad + if state.is_main_process: + tensor = torch.arange(state.num_processes + 1).to(state.device) + else: + tensor = torch.arange(state.num_processes).to(state.device) + padded_tensor = pad_across_processes(tensor) + assert padded_tensor.shape == torch.Size([state.num_processes + 1]) + if not state.is_main_process: + assert padded_tensor.tolist() == list(range(0, state.num_processes)) + [0] + + +def test_reduce_sum(state): + # For now runs on only two processes + if state.num_processes != 2: + return + tensor = create_tensor(state) + reduced_tensor = reduce(tensor, "sum") + truth_tensor = torch.tensor([4.0, 6]).to(state.device) + assert torch.allclose(reduced_tensor, truth_tensor), f"{reduced_tensor} != {truth_tensor}" + + +def test_reduce_mean(state): + # For now runs on only two processes + if state.num_processes != 2: + return + tensor = create_tensor(state) + reduced_tensor = reduce(tensor, "mean") + truth_tensor = torch.tensor([2.0, 3]).to(state.device) + assert torch.allclose(reduced_tensor, truth_tensor), f"{reduced_tensor} != {truth_tensor}" + + +def test_reduce_max(state): + # For now runs on only two processes + if state.num_processes != 2: + return + tensor = create_tensor(state) + reduced_tensor = reduce(tensor, "max") + truth_tensor = torch.tensor([3.0, 4]).to(state.device) + assert torch.allclose(reduced_tensor, truth_tensor), f"{reduced_tensor} != {truth_tensor}" + + +def test_op_checker(state): + # Must be in a distributed state, and gathering is currently not supported in TorchXLA. + if state.distributed_type in [DistributedType.NO, DistributedType.XLA]: + return + state.debug = True + # `pad_across_processes` + if state.process_index == 0: + data = {"tensor": torch.tensor([[0.0, 1, 2, 3, 4]]).to(state.device)} + else: + data = {"tensor": torch.tensor([[[0.0, 1, 2, 3, 4, 5]]]).to(state.device)} + + with assert_exception(DistributedOperationException): + pad_across_processes(data, dim=0) + + # `reduce` + if state.process_index == 0: + data = {"tensor": torch.tensor([[0.0, 1, 2, 3, 4]]).to(state.device)} + else: + data = {"tensor": torch.tensor([[[0.0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]]).to(state.device)} + + with assert_exception(DistributedOperationException): + reduce(data) + + # `broadcast` + if state.process_index == 0: + data = {"tensor": torch.tensor([[0.0, 1, 2, 3, 4]]).to(state.device)} + else: + data = {"tensor": torch.tensor([[[0.0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]]).to(state.device)} + + with assert_exception(DistributedOperationException): + broadcast(data) + + state.debug = False + + +def test_copy_tensor_to_devices(state): + if state.distributed_type not in [DistributedType.MULTI_GPU, DistributedType.XLA]: + return + if state.is_main_process: + tensor = torch.tensor([1, 2, 3], dtype=torch.int).to(state.device) + else: + tensor = None + tensor = copy_tensor_to_devices(tensor) + assert torch.allclose(tensor, torch.tensor([1, 2, 3], dtype=torch.int, device=state.device)) + + +def _mp_fn(index): + # For xla_spawn (TPUs) + main() + + +def main(): + state = PartialState() + state.print(f"State: {state}") + state.print("testing gather") + test_gather(state) + state.print("testing gather_object") + test_gather_object(state) + state.print("testing gather non-contiguous") + test_gather_non_contiguous(state) + state.print("testing broadcast") + test_broadcast(state) + state.print("testing pad_across_processes") + test_pad_across_processes(state) + state.print("testing reduce_sum") + test_reduce_sum(state) + state.print("testing reduce_mean") + test_reduce_mean(state) + state.print("testing reduce_max") + test_reduce_max(state) + state.print("testing op_checker") + test_op_checker(state) + state.print("testing sending tensors across devices") + test_copy_tensor_to_devices(state) + state.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_script.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_script.py new file mode 100644 index 0000000000000000000000000000000000000000..e53e12a18fa79c221640e2e73bdf83c552f66bfa --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_script.py @@ -0,0 +1,909 @@ +#!/usr/bin/env python + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import io +import math +import time +from copy import deepcopy +from pathlib import Path + +import numpy as np +import torch +from torch.utils.data import DataLoader, Dataset + +from accelerate import Accelerator +from accelerate.data_loader import SeedableRandomSampler, prepare_data_loader +from accelerate.state import AcceleratorState +from accelerate.test_utils import RegressionDataset, RegressionModel, are_the_same_tensors +from accelerate.utils import ( + DataLoaderConfiguration, + DistributedType, + gather, + gather_object, + is_bf16_available, + is_cuda_available, + is_datasets_available, + is_fp16_available, + is_hpu_available, + is_mps_available, + is_pytest_available, + set_seed, + synchronize_rng_states, +) + + +if is_hpu_available(): + ATOL = 1e-3 + RTOL = 1e-3 +else: + ATOL = 1e-6 + RTOL = 1e-6 + + +def generate_baseline_dataloader(train_set, generator, batch_size, use_seedable_sampler=False): + "Creates a dataloader that can also use the `SeedableRandomSampler`" + if use_seedable_sampler: + # The SeedableRandomSampler is needed during distributed setups + # for full reproducibility across processes with the `DataLoader` + sampler = SeedableRandomSampler( + generator=generator, + data_source=train_set, + num_samples=len(train_set), + ) + return DataLoader(train_set, batch_size=batch_size, sampler=sampler) + else: + return DataLoader(train_set, batch_size=batch_size, shuffle=True, generator=generator) + + +def print_main(state): + print(f"Printing from the main process {state.process_index}") + + +def print_local_main(state): + print(f"Printing from the local main process {state.local_process_index}") + + +def print_last(state): + print(f"Printing from the last process {state.process_index}") + + +def print_on(state, process_idx): + print(f"Printing from process {process_idx}: {state.process_index}") + + +def process_execution_check(): + accelerator = Accelerator() + num_processes = accelerator.num_processes + # Test main_process_first context manager + path = Path("check_main_process_first.txt") + with accelerator.main_process_first(): + if accelerator.is_main_process: + time.sleep(0.1) # ensure main process takes longest + with open(path, "a+") as f: + f.write("Currently in the main process\n") + else: + with open(path, "a+") as f: + f.write("Now on another process\n") + accelerator.wait_for_everyone() + + if accelerator.is_main_process: + with open(path) as f: + text = "".join(f.readlines()) + try: + assert text.startswith("Currently in the main process\n"), "Main process was not first" + if num_processes > 1: + assert text.endswith("Now on another process\n"), "Main process was not first" + assert text.count("Now on another process\n") == accelerator.num_processes - 1, ( + f"Only wrote to file {text.count('Now on another process') + 1} times, not {accelerator.num_processes}" + ) + except AssertionError: + path.unlink() + raise + + if accelerator.is_main_process and path.exists(): + path.unlink() + accelerator.wait_for_everyone() + # Test the decorators + f = io.StringIO() + with contextlib.redirect_stdout(f): + accelerator.on_main_process(print_main)(accelerator.state) + result = f.getvalue().rstrip() + if accelerator.is_main_process: + assert result == "Printing from the main process 0", f"{result} != Printing from the main process 0" + else: + assert f.getvalue().rstrip() == "", f'{result} != ""' + f.truncate(0) + f.seek(0) + + with contextlib.redirect_stdout(f): + accelerator.on_local_main_process(print_local_main)(accelerator.state) + if accelerator.is_local_main_process: + assert f.getvalue().rstrip() == "Printing from the local main process 0" + else: + assert f.getvalue().rstrip() == "" + f.truncate(0) + f.seek(0) + + with contextlib.redirect_stdout(f): + accelerator.on_last_process(print_last)(accelerator.state) + if accelerator.is_last_process: + assert f.getvalue().rstrip() == f"Printing from the last process {accelerator.state.num_processes - 1}" + else: + assert f.getvalue().rstrip() == "" + f.truncate(0) + f.seek(0) + + for process_idx in range(num_processes): + with contextlib.redirect_stdout(f): + accelerator.on_process(print_on, process_index=process_idx)(accelerator.state, process_idx) + if accelerator.process_index == process_idx: + assert f.getvalue().rstrip() == f"Printing from process {process_idx}: {accelerator.process_index}" + else: + assert f.getvalue().rstrip() == "" + f.truncate(0) + f.seek(0) + + +def init_state_check(): + # Test we can instantiate this twice in a row. + state = AcceleratorState() + if state.local_process_index == 0: + print("Testing, testing. 1, 2, 3.") + print(state) + + +def rng_sync_check(): + state = AcceleratorState() + synchronize_rng_states(["torch"]) + assert are_the_same_tensors(torch.get_rng_state()), "RNG states improperly synchronized on CPU." + if state.distributed_type == DistributedType.MULTI_GPU: + synchronize_rng_states(["cuda"]) + assert are_the_same_tensors(torch.cuda.get_rng_state()), "RNG states improperly synchronized on GPU." + elif state.distributed_type == DistributedType.MULTI_XPU: + synchronize_rng_states(["xpu"]) + assert are_the_same_tensors(torch.xpu.get_rng_state()), "RNG states improperly synchronized on XPU." + generator = torch.Generator() + synchronize_rng_states(["generator"], generator=generator) + assert are_the_same_tensors(generator.get_state()), "RNG states improperly synchronized in generator." + + if state.local_process_index == 0: + print("All rng are properly synched.") + + +def dl_preparation_check(): + state = AcceleratorState() + length = 32 * state.num_processes + + dl = DataLoader(range(length), batch_size=8) + dl = prepare_data_loader(dl, state.device, state.num_processes, state.process_index, put_on_device=True) + result = [] + for batch in dl: + result.append(gather(batch)) + result = torch.cat(result) + + assert torch.equal(result.cpu(), torch.arange(0, length).long()), "Wrong non-shuffled dataloader result." + + dl = DataLoader(range(length), batch_size=8) + dl = prepare_data_loader( + dl, + state.device, + state.num_processes, + state.process_index, + put_on_device=True, + split_batches=True, + ) + result = [] + for batch in dl: + result.append(gather(batch)) + result = torch.cat(result) + assert torch.equal(result.cpu(), torch.arange(0, length).long()), "Wrong non-shuffled dataloader result." + + if state.process_index == 0: + print("Non-shuffled dataloader passing.") + + dl = DataLoader(range(length), batch_size=8, shuffle=True) + dl = prepare_data_loader(dl, state.device, state.num_processes, state.process_index, put_on_device=True) + result = [] + for batch in dl: + result.append(gather(batch)) + result = torch.cat(result).tolist() + result.sort() + assert result == list(range(length)), "Wrong shuffled dataloader result." + + dl = DataLoader(range(length), batch_size=8, shuffle=True) + dl = prepare_data_loader( + dl, + state.device, + state.num_processes, + state.process_index, + put_on_device=True, + split_batches=True, + ) + result = [] + for batch in dl: + result.append(gather(batch)) + result = torch.cat(result).tolist() + result.sort() + assert result == list(range(length)), "Wrong shuffled dataloader result." + + if state.local_process_index == 0: + print("Shuffled dataloader passing.") + + +def central_dl_preparation_check(): + state = AcceleratorState() + length = 32 * state.num_processes + + dl = DataLoader(range(length), batch_size=8) + dl = prepare_data_loader( + dl, state.device, state.num_processes, state.process_index, put_on_device=True, dispatch_batches=True + ) + result = [] + for batch in dl: + result.append(gather(batch)) + result = torch.cat(result) + assert torch.equal(result.cpu(), torch.arange(0, length).long()), "Wrong non-shuffled dataloader result." + + dl = DataLoader(range(length), batch_size=8) + dl = prepare_data_loader( + dl, + state.device, + state.num_processes, + state.process_index, + put_on_device=True, + split_batches=True, + dispatch_batches=True, + ) + result = [] + for batch in dl: + result.append(gather(batch)) + result = torch.cat(result) + assert torch.equal(result.cpu(), torch.arange(0, length).long()), "Wrong non-shuffled dataloader result." + + if state.process_index == 0: + print("Non-shuffled central dataloader passing.") + + dl = DataLoader(range(length), batch_size=8, shuffle=True) + dl = prepare_data_loader( + dl, state.device, state.num_processes, state.process_index, put_on_device=True, dispatch_batches=True + ) + result = [] + for batch in dl: + result.append(gather(batch)) + result = torch.cat(result).tolist() + result.sort() + assert result == list(range(length)), "Wrong shuffled dataloader result." + + dl = DataLoader(range(length), batch_size=8, shuffle=True) + dl = prepare_data_loader( + dl, + state.device, + state.num_processes, + state.process_index, + put_on_device=True, + split_batches=True, + dispatch_batches=True, + ) + result = [] + for batch in dl: + result.append(gather(batch)) + result = torch.cat(result).tolist() + result.sort() + assert result == list(range(length)), "Wrong shuffled dataloader result." + + if state.local_process_index == 0: + print("Shuffled central dataloader passing.") + + +def custom_sampler_check(): + state = AcceleratorState() + + class CustomDataset(Dataset): + def __init__(self, data): + self.data = data + + def __len__(self): + return len(self.data) + + def __getitem__(self, index): + return self.data[index] + + class CustomBatchSampler: + def __init__(self, dataset_length: int, batch_size: int, shuffle: bool = True): + self.batch_size = batch_size + self.data_index = np.arange(dataset_length) + self.shuffle = shuffle + + def __iter__(self): + num_batches = len(self) + if self.shuffle: + index = np.random.permutation(self.data_index) + else: + index = self.data_index + output = np.array_split(index, num_batches) + yield from output + + def __len__(self): + return math.ceil(len(self.data_index) / self.batch_size) + + dataset = CustomDataset(range(32 * state.num_processes)) + sampler = CustomBatchSampler(len(dataset), batch_size=8) + dl = DataLoader(dataset, batch_sampler=sampler) + dl = prepare_data_loader(dl, state.device, state.num_processes, state.process_index) + # We need just ensure that `dl.batch_sampler` (or `dl.batch_sampler.batch_sampler` is indeed the old batch sampler + if hasattr(dl.batch_sampler, "batch_sampler"): + assert isinstance(dl.batch_sampler.batch_sampler, CustomBatchSampler), ( + "Custom sampler was changed after calling `prepare_data_loader`" + ) + else: + assert isinstance(dl.batch_sampler, CustomBatchSampler), ( + "Custom sampler was changed after calling `prepare_data_loader`" + ) + + +def check_seedable_sampler(): + # Set seed + set_seed(42) + train_set = RegressionDataset(length=10, seed=42) + train_dl = DataLoader(train_set, batch_size=2, shuffle=True) + + config = DataLoaderConfiguration(use_seedable_sampler=True) + accelerator = Accelerator(dataloader_config=config) + train_dl = accelerator.prepare(train_dl) + original_items = [] + for _ in range(3): + for batch in train_dl: + original_items.append(batch["x"]) + original_items = torch.cat(original_items) + + # Set seed again and the epoch + set_seed(42) + train_dl.set_epoch(0) + new_items = [] + for _ in range(3): + for batch in train_dl: + new_items.append(batch["x"]) + new_items = torch.cat(new_items) + assert torch.allclose(original_items, new_items), "Did not obtain the same items with the same seed and epoch." + + +def check_seedable_sampler_in_batch_sampler_shard(): + set_seed(42) + + config = DataLoaderConfiguration(use_seedable_sampler=True) + accelerator = Accelerator(dataloader_config=config) + assert accelerator.num_processes > 1, "This test requires more than one process." + + dataloader = DataLoader(list(range(10)), batch_size=1, shuffle=True) + prepared_data_loader = prepare_data_loader( + dataloader=dataloader, + use_seedable_sampler=True, + ) + + target_sampler = prepared_data_loader.batch_sampler.batch_sampler.sampler + assert isinstance(target_sampler, SeedableRandomSampler), ( + "Sampler in BatchSamplerShard is not SeedableRandomSampler." + ) + + +def check_seedable_sampler_with_data_seed(): + # Set seed + set_seed(42) + data_seed = 42 + train_set = RegressionDataset(length=10, seed=42) + train_dl = DataLoader(train_set, batch_size=2, shuffle=True) + + config = DataLoaderConfiguration(use_seedable_sampler=True, data_seed=data_seed) + accelerator = Accelerator(dataloader_config=config) + prepared_dl = accelerator.prepare(train_dl) + original_items = [] + for _ in range(3): + for batch in prepared_dl: + original_items.append(batch["x"]) + original_items = torch.cat(original_items) + + # Set new data seed + config.data_seed = 43 + accelerator = Accelerator(dataloader_config=config) + prepared_dl = accelerator.prepare(train_dl) + new_items = [] + for _ in range(3): + for batch in prepared_dl: + new_items.append(batch["x"]) + new_items = torch.cat(new_items) + assert not torch.allclose(original_items, new_items), "Obtained the same items with different data seed." + + +def mock_training(length, batch_size, generator, use_seedable_sampler=False): + set_seed(42) + generator.manual_seed(42) + train_set = RegressionDataset(length=length, seed=42) + + train_dl = generate_baseline_dataloader(train_set, generator, batch_size, use_seedable_sampler) + model = RegressionModel() + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + for epoch in range(3): + for batch in train_dl: + model.zero_grad() + output = model(batch["x"]) + loss = torch.nn.functional.mse_loss(output, batch["y"]) + loss.backward() + optimizer.step() + return train_set, model + + +def training_check(use_seedable_sampler=False): + state = AcceleratorState() + generator = torch.Generator() + batch_size = 8 + length = batch_size * 4 * state.num_processes + + train_set, old_model = mock_training(length, batch_size * state.num_processes, generator, use_seedable_sampler) + assert are_the_same_tensors(old_model.a), "Did not obtain the same model on both processes." + assert are_the_same_tensors(old_model.b), "Did not obtain the same model on both processes." + + accelerator = Accelerator() + train_dl = generate_baseline_dataloader(train_set, generator, batch_size, use_seedable_sampler) + model = RegressionModel() + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + + train_dl, model, optimizer = accelerator.prepare(train_dl, model, optimizer) + set_seed(42) + generator.manual_seed(42) + for _ in range(3): + for batch in train_dl: + model.zero_grad() + output = model(batch["x"]) + loss = torch.nn.functional.mse_loss(output, batch["y"]) + accelerator.backward(loss) + optimizer.step() + + model = accelerator.unwrap_model(model).cpu() + torch.testing.assert_close( + old_model.a, + model.a, + atol=ATOL, + rtol=RTOL, + msg=lambda msg: f"Did not obtain the same model on CPU or distributed training.\n{msg}", + ) + torch.testing.assert_close( + old_model.b, + model.b, + atol=ATOL, + rtol=RTOL, + msg=lambda msg: f"Did not obtain the same model on CPU or distributed training.\n{msg}", + ) + + accelerator.print("Training yielded the same results on one CPU or distributed setup with no batch split.") + + dataloader_config = DataLoaderConfiguration(split_batches=True, use_seedable_sampler=use_seedable_sampler) + accelerator = Accelerator(dataloader_config=dataloader_config) + train_dl = generate_baseline_dataloader( + train_set, generator, batch_size * state.num_processes, use_seedable_sampler + ) + model = RegressionModel() + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + + train_dl, model, optimizer = accelerator.prepare(train_dl, model, optimizer) + set_seed(42) + generator.manual_seed(42) + for _ in range(3): + for batch in train_dl: + model.zero_grad() + output = model(batch["x"]) + loss = torch.nn.functional.mse_loss(output, batch["y"]) + accelerator.backward(loss) + optimizer.step() + + model = accelerator.unwrap_model(model).cpu() + torch.testing.assert_close( + old_model.a, + model.a, + atol=ATOL, + rtol=RTOL, + msg=lambda msg: f"Did not obtain the same model on CPU or distributed training.\n{msg}", + ) + torch.testing.assert_close( + old_model.b, + model.b, + atol=ATOL, + rtol=RTOL, + msg=lambda msg: f"Did not obtain the same model on CPU or distributed training.\n{msg}", + ) + + accelerator.print("Training yielded the same results on one CPU or distributed setup with batch split.") + + # FP32 wrapper check + if is_cuda_available() or is_mps_available(): + # Mostly a test that model.forward will have autocast when running unwrap_model(model, keep_fp32_wrapper=True) + print("Keep fp32 wrapper check.") + AcceleratorState._reset_state() + accelerator = Accelerator(mixed_precision="fp16") + + model = torch.nn.Linear(2, 4) + model = accelerator.prepare(model) + model_with_fp32_wrapper = accelerator.unwrap_model(model, keep_fp32_wrapper=True) + + # Run forward with fp16 as input. + # When the model is with mixed precision wrapper, no error will be raised. + input_tensor = torch.Tensor([1, 2]).to(dtype=torch.float16, device=accelerator.device) + output = model_with_fp32_wrapper(input_tensor) + + # BF16 support + if is_bf16_available(): + # Mostly a test that BF16 doesn't crash as the operation inside the model is not converted to BF16 + print("BF16 training check.") + AcceleratorState._reset_state() + dataloader_config = DataLoaderConfiguration(use_seedable_sampler=use_seedable_sampler) + accelerator = Accelerator(mixed_precision="bf16", dataloader_config=dataloader_config) + train_dl = generate_baseline_dataloader(train_set, generator, batch_size, use_seedable_sampler) + model = RegressionModel() + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + + train_dl, model, optimizer = accelerator.prepare(train_dl, model, optimizer) + set_seed(42) + generator.manual_seed(42) + for _ in range(3): + for batch in train_dl: + model.zero_grad() + output = model(batch["x"]) + loss = torch.nn.functional.mse_loss(output, batch["y"]) + accelerator.backward(loss) + optimizer.step() + + model = accelerator.unwrap_model(model).cpu() + torch.testing.assert_close( + old_model.a, + model.a, + atol=ATOL, + rtol=RTOL, + msg=lambda msg: f"Did not obtain the same model on CPU or distributed training.\n{msg}", + ) + torch.testing.assert_close( + old_model.b, + model.b, + atol=ATOL, + rtol=RTOL, + msg=lambda msg: f"Did not obtain the same model on CPU or distributed training.\n{msg}", + ) + + # FP16 support (HPU fp16 model seems to be off by 10% from the CPU, which is a lot of numerical error) + if is_fp16_available() and not is_hpu_available(): + # Mostly a test that FP16 doesn't crash as the operation inside the model is not converted to FP16 + print("FP16 training check.") + AcceleratorState._reset_state() + dataloader_config = DataLoaderConfiguration(use_seedable_sampler=use_seedable_sampler) + accelerator = Accelerator(mixed_precision="fp16", dataloader_config=dataloader_config) + train_dl = generate_baseline_dataloader(train_set, generator, batch_size, use_seedable_sampler) + model = RegressionModel() + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + + train_dl, model, optimizer = accelerator.prepare(train_dl, model, optimizer) + set_seed(42) + generator.manual_seed(42) + for _ in range(3): + for batch in train_dl: + model.zero_grad() + output = model(batch["x"]) + loss = torch.nn.functional.mse_loss(output, batch["y"]) + accelerator.backward(loss) + optimizer.step() + + model = accelerator.unwrap_model(model).cpu() + torch.testing.assert_close( + old_model.a, + model.a, + atol=ATOL, + rtol=RTOL, + msg=lambda msg: f"Did not obtain the same model on CPU or distributed training.\n{msg}", + ) + torch.testing.assert_close( + old_model.b, + model.b, + atol=ATOL, + rtol=RTOL, + msg=lambda msg: f"Did not obtain the same model on CPU or distributed training.\n{msg}", + ) + + +def test_split_between_processes_dataset(datasets_Dataset): + state = AcceleratorState() + data = datasets_Dataset.from_list([dict(k=v) for v in range(2 * state.num_processes)]) + with state.split_between_processes(data, apply_padding=False) as results: + assert len(results) == 2, ( + f"Each process did not have two items. Process index: {state.process_index}; Length: {len(results)}" + ) + + data = datasets_Dataset.from_list([dict(k=v) for v in range(2 * state.num_processes - 1)]) + with state.split_between_processes(data, apply_padding=False) as results: + if state.is_last_process: + assert len(results) == 1, ( + f"Last process did not receive a single item. Process index: {state.process_index}; Length: {len(results)}" + ) + else: + assert len(results) == 2, ( + f"One of the intermediate processes did not receive two items. Process index: {state.process_index}; Length: {len(results)}" + ) + state.wait_for_everyone() + + odd_data = datasets_Dataset.from_list([dict(k=v) for v in range(2 * state.num_processes - 1)]) + even_data = datasets_Dataset.from_list([dict(k=v) for v in range(2 * state.num_processes)]) + + for data in [odd_data, even_data]: + expected_output = data["k"] + + with state.split_between_processes(data, apply_padding=True) as results: + if state.num_processes == 1: + assert len(results) == len(data), ( + f"Single process did not receive all items. Process index: {state.process_index}; Length: {len(results)}" + ) + else: + assert len(results) == 2, ( + f"Each process did not have two items. Process index: {state.process_index}; Length: {len(results)}" + ) + + results_per_process = [] + for result in results: + results_per_process.append(result) + + state.wait_for_everyone() + + gathered_results = gather_object(results_per_process) + output = [r["k"] for r in gathered_results[: len(data)]] + + assert expected_output == output, f"Gathered results is incorrect. Expected: {expected_output}; Got: {output}" + + +def test_split_between_processes_list(): + state = AcceleratorState() + data = list(range(0, 2 * state.num_processes)) + with state.split_between_processes(data) as results: + assert len(results) == 2, ( + f"Each process did not have two items. Process index: {state.process_index}; Length: {len(results)}" + ) + state.wait_for_everyone() + + even_data = list(range(0, (2 * state.num_processes))) + odd_data = list(range(0, (2 * state.num_processes) - 1)) + for data in [odd_data, even_data]: + expected_output = data + + with state.split_between_processes(data, apply_padding=True) as results: + num_samples_per_device = math.ceil(len(data) / state.num_processes) + # Test all processes gets the correct number of item(s) + assert len(results) == num_samples_per_device, ( + f"Process {state.device} did not get the correct number of item(s). Process index: {state.process_index}; Length: {len(results)}" + ) + + results_per_process = [] + for result in results: + results_per_process.append(result) + + state.wait_for_everyone() + + gathered_results = gather_object(results_per_process) + output = gathered_results[: len(data)] + + assert expected_output == output, f"Gathered results is incorrect. Expected: {expected_output}; Got: {output}" + + +def test_split_between_processes_nested_dict(): + state = AcceleratorState() + a = [1, 2, 3, 4, 5, 6, 7, 8] + b = ["a", "b", "c", "d", "e", "f", "g", "h"] + c = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8]) + if state.num_processes in (1, 2, 4): + data = {"a": a, "b": b, "c": c} + data_copy = deepcopy(data) + with state.split_between_processes(data) as results: + if state.process_index == 0: + assert results["a"] == data_copy["a"][: 8 // state.num_processes] + elif state.num_processes == 2: + assert results["a"] == data_copy["a"][4:] + elif state.process_index == 3: + # We return a list each time + assert results["a"] == data_copy["a"][-2:], f"Expected: {data_copy['a'][-2]}, Actual: {results['a']}" + if state.process_index == 0: + assert results["b"] == data_copy["b"][: 8 // state.num_processes] + elif state.num_processes == 2: + assert results["b"] == data_copy["b"][4:] + elif state.process_index == 3: + assert results["b"] == data_copy["b"][-2:] + if state.process_index == 0: + assert torch.allclose(results["c"], data_copy["c"][: 8 // state.num_processes]), ( + f"Did not obtain expected values on process 0, expected `{data['c'][: 8 // state.num_processes]}`, received: {results['c']}" + ) + elif state.num_processes == 2: + assert torch.allclose(results["c"], data_copy["c"][4:]), ( + f"Did not obtain expected values on process 2, expected `{data['c'][4:]}`, received: {results['c']}" + ) + elif state.process_index == 3: + assert torch.allclose(results["c"], data_copy["c"][-2:]), ( + f"Did not obtain expected values on process 4, expected `{data['c'][-2:]}`, received: {results['c']}" + ) + + state.wait_for_everyone() + + +def test_split_between_processes_tensor(): + state = AcceleratorState() + if state.num_processes > 1: + data = torch.tensor([[0, 1, 2, 3], [4, 5, 6, 7]]).to(state.device) + with state.split_between_processes(data) as results: + if state.process_index == 0: + expected = torch.tensor([[0, 1, 2, 3]]).to(state.device) + else: + expected = torch.tensor([[4, 5, 6, 7]]).to(state.device) + torch.testing.assert_close(results, expected) + state.wait_for_everyone() + + even_data = torch.tensor([[i] for i in range(2 * state.num_processes)]).to(state.device) + odd_data = torch.tensor([[i] for i in range(2 * state.num_processes - 1)]).to(state.device) + for data in [even_data, odd_data]: + expected_output = [torch.tensor(i) for i in data.tolist()] + + with state.split_between_processes(data, apply_padding=True) as results: + num_samples_per_device = math.ceil(len(data) / state.num_processes) + assert len(results) == num_samples_per_device, ( + f"Process {state.device} did not get the correct number of item(s). Process index: {state.process_index}; Length: {len(results)}" + ) + results_per_process = [] + for result in results: + results_per_process.append(result.to("cpu")) + + state.wait_for_everyone() + + gathered_results = gather_object(results_per_process) + output = gathered_results[: len(data)] + + assert expected_output == output, f"Gathered results is incorrect. Expected: {expected_output}; Got: {output}" + + +def test_split_between_processes_evenly(): + state = AcceleratorState() + if state.num_processes in (1, 2, 4, 8): + data = list(range(17)) + num_samples_per_process = len(data) // state.num_processes + num_extras = len(data) % state.num_processes + with state.split_between_processes(data) as results: + if state.process_index < num_extras: + assert len(results) == num_samples_per_process + 1, ( + f"Each Process should have even elements. Expected: {num_samples_per_process + 1}, Actual: {len(results)}" + ) + else: + assert len(results) == num_samples_per_process, ( + f"Each Process should have even elements. Expected: {num_samples_per_process}, Actual: {len(results)}" + ) + state.wait_for_everyone() + + +def test_trigger(): + accelerator = Accelerator() + # should start with being false + assert accelerator.check_trigger() is False + + # set a breakpoint on the main process + if accelerator.is_main_process: + accelerator.set_trigger() + + # check it's been activated across all processes + # calls `all_reduce` and triggers a sync + assert accelerator.check_trigger() is True + + # check it's been reset after the sync + assert accelerator.check_trigger() is False + + +def test_reinstantiated_state(): + import pytest + + AcceleratorState._reset_state() + simple_model = torch.nn.Linear(1, 1) + # First define an accelerator + accelerator = Accelerator() + # Then call `reset_state`, breaking the state existing in the accelerator + AcceleratorState._reset_state() + # Now try and prepare a simple model, should raise the custom error early + with pytest.raises(AttributeError) as cm: + accelerator.prepare(simple_model) + assert "`AcceleratorState` object has no attribute" in str(cm.value.args[0]) + assert "This happens if `AcceleratorState._reset_state()`" in str(cm.value.args[0]) + + +def main(): + accelerator = Accelerator() + state = accelerator.state + if state.local_process_index == 0: + print("**Initialization**") + init_state_check() + state.wait_for_everyone() + + if state.distributed_type == DistributedType.MULTI_GPU: + num_processes_per_node = torch.cuda.device_count() + else: + num_processes_per_node = state.num_processes + + # We only run this test on non-multinode + if num_processes_per_node == state.num_processes: + if state.process_index == 0: + print("\n**Test process execution**") + process_execution_check() + + if state.process_index == 0: + print("\n**Test split between processes as a list**") + test_split_between_processes_list() + + if state.process_index == 0: + print("\n**Test split between processes as a dict**") + test_split_between_processes_nested_dict() + + if state.process_index == 0: + print("\n**Test split between processes as a tensor**") + test_split_between_processes_tensor() + + if state.process_index == 0: + print("\n**Test split between processes evenly**") + test_split_between_processes_evenly() + + if state.process_index == 0: + print("\n**Test split between processes as a datasets.Dataset**") + if is_datasets_available(): + from datasets import Dataset as datasets_Dataset + + test_split_between_processes_dataset(datasets_Dataset) + else: + print("Skipped because Hugging Face datasets is not available") + + if state.local_process_index == 0: + print("\n**Test random number generator synchronization**") + rng_sync_check() + + if state.local_process_index == 0: + print("\n**DataLoader integration test**") + dl_preparation_check() + if state.distributed_type != DistributedType.XLA: + central_dl_preparation_check() + custom_sampler_check() + check_seedable_sampler() + check_seedable_sampler_with_data_seed() + + if state.num_processes > 1: + check_seedable_sampler_in_batch_sampler_shard() + + # Trainings are not exactly the same in DeepSpeed and CPU mode + if state.distributed_type == DistributedType.DEEPSPEED: + return + + if state.local_process_index == 0: + print("\n**Training integration test**") + training_check(use_seedable_sampler=False) + training_check(use_seedable_sampler=True) + + if state.local_process_index == 0: + print("\n**Breakpoint trigger test**") + test_trigger() + + if is_pytest_available(): + if state.local_process_index == 0: + print("\n**Test reinstantiated state**") + test_reinstantiated_state() + + state.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_sync.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_sync.py new file mode 100644 index 0000000000000000000000000000000000000000..efe482649c39ca4626ed30d287081c7027ea4d9d --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/scripts/test_sync.py @@ -0,0 +1,414 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from copy import deepcopy + +import torch +import torch.nn.functional as F +from torch.optim import AdamW +from torch.optim.lr_scheduler import LambdaLR +from torch.utils.data import DataLoader + +from accelerate.accelerator import Accelerator, DataLoaderConfiguration, GradientAccumulationPlugin +from accelerate.state import GradientState +from accelerate.test_utils import RegressionDataset, RegressionModel +from accelerate.utils import DistributedType, set_seed + + +def check_model_parameters(model_a, model_b, did_step, iteration, **kwargs): + # kwargs (e.g. a loosened rtol) only applies to the in-sync check; the not-in-sync check stays strict + for param, grad_param in zip(model_a.parameters(), model_b.parameters()): + if not param.requires_grad: + continue + if not did_step: + # Grads should not be in sync + assert torch.allclose(param.grad, grad_param.grad) is False, ( + f"Gradients in sync when they should not be at iteration {iteration}:\nmodel_a grad ({param.grad}) == model_b grad ({grad_param.grad})" + ) + else: + # Grads should be in sync + assert torch.allclose(param.grad, grad_param.grad, **kwargs) is True, ( + f"Gradients not in sync when they should be at iteration {iteration}:\nmodel_a grad ({param.grad}) != model_b grad ({grad_param.grad})" + ) + + +def step_model(model, input, target, accelerator, do_backward=True): + model.train() + output = model(input) + loss = F.mse_loss(output, target.to(output.device)) + if not do_backward: + loss /= accelerator.gradient_accumulation_steps + loss.backward() + else: + accelerator.backward(loss) + + +def get_training_setup(accelerator, sched=False): + "Returns everything needed to perform basic training" + set_seed(42) + model = RegressionModel() + ddp_model = deepcopy(model) + dset = RegressionDataset(length=80) + dataloader = DataLoader(dset, batch_size=16) + model.to(accelerator.device) + if sched: + opt = AdamW(params=model.parameters(), lr=1e-3) + ddp_opt = AdamW(params=ddp_model.parameters(), lr=1e-3) + sched = LambdaLR(opt, lr_lambda=lambda epoch: epoch**0.65) + ddp_sched = LambdaLR(ddp_opt, lr_lambda=lambda epoch: epoch**0.65) + # Make a copy of `model` + if sched: + ddp_model, ddp_opt, ddp_sched, dataloader = accelerator.prepare(ddp_model, ddp_opt, ddp_sched, dataloader) + else: + ddp_model, dataloader = accelerator.prepare(ddp_model, dataloader) + if sched: + return (model, opt, sched, dataloader, ddp_model, ddp_opt, ddp_sched) + return model, ddp_model, dataloader + + +def test_noop_sync(accelerator): + # Test when on a single CPU or GPU that the context manager does nothing + model, ddp_model, dataloader = get_training_setup(accelerator) + # Use a single batch + ddp_input, ddp_target = next(iter(dataloader)).values() + for iteration in range(3): + # Gather the distributed inputs and targs for the base model + input, target = accelerator.gather((ddp_input, ddp_target)) + input, target = input.to(accelerator.device), target.to(accelerator.device) + # Perform our initial ground truth step in non "DDP" + step_model(model, input, target, accelerator) + # Do "gradient accumulation" (noop) + if iteration % 2 == 0: + # Accumulate grads locally + with accelerator.no_sync(ddp_model): + step_model(ddp_model, ddp_input, ddp_target, accelerator) + else: + # Sync grads + step_model(ddp_model, ddp_input, ddp_target, accelerator) + + # Since `no_sync` is a noop, `ddp_model` and `model` grads should always be in sync + check_model_parameters(model, ddp_model, True, iteration) + for param, ddp_param in zip(model.parameters(), ddp_model.parameters()): + if not param.requires_grad: + continue + assert torch.allclose(param.grad, ddp_param.grad), ( + f"Gradients not in sync when they should be:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})" + ) + + # Shuffle ddp_input on each iteration + torch.manual_seed(1337 + iteration) + ddp_input = ddp_input[torch.randperm(len(ddp_input))] + + +def test_distributed_sync(accelerator): + # Test on distributed setup that context manager behaves properly + model, ddp_model, dataloader = get_training_setup(accelerator) + # Use a single batch + ddp_input, ddp_target = next(iter(dataloader)).values() + for iteration in range(3): + # Gather the distributed inputs and targs for the base model + input, target = accelerator.gather((ddp_input, ddp_target)) + input, target = input.to(accelerator.device), target.to(accelerator.device) + # Perform our initial ground truth step in non "DDP" + step_model(model, input, target, accelerator) + # Do "gradient accumulation" (noop) + if iteration % 2 == 0: + # Accumulate grads locally + with accelerator.no_sync(ddp_model): + step_model(ddp_model, ddp_input, ddp_target, accelerator) + else: + # Sync grads + step_model(ddp_model, ddp_input, ddp_target, accelerator) + + # DDP model and model should only be in sync when not (iteration % 2 == 0) + for param, ddp_param in zip(model.parameters(), ddp_model.parameters()): + if not param.requires_grad: + continue + if iteration % 2 == 0: + # Grads should not be in sync + assert torch.allclose(param.grad, ddp_param.grad) is False, ( + f"Gradients in sync when they should not be:\nModel grad ({param.grad}) == DDP grad ({ddp_param.grad})" + ) + else: + # Grads should be in sync + assert torch.allclose(param.grad, ddp_param.grad) is True, ( + f"Gradients not in sync when they should be:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})" + ) + + # Shuffle ddp_input on each iteration + torch.manual_seed(1337 + iteration) + ddp_input = ddp_input[torch.randperm(len(ddp_input))] + + +def test_distributed_sync_multiple_fwd(accelerator): + # Test on distributed setup that context manager behaves properly when used with multiple forwards followed by multiple backwards + model, ddp_model, dataloader = get_training_setup(accelerator) + # Do multiple forwards + losses = [] + num_iterations = 3 + for iteration in range(num_iterations): + ddp_input, ddp_target = next(iter(dataloader)).values() + + # Gather the distributed inputs and targs for the base model + input, target = accelerator.gather((ddp_input, ddp_target)) + input, target = input.to(accelerator.device), target.to(accelerator.device) + + # Perform our initial ground truth step in non "DDP" + step_model(model, input, target, accelerator) + + # Accumulate grads locally + with accelerator.no_sync(ddp_model): + ddp_output = ddp_model(ddp_input) + loss = F.mse_loss(ddp_output, ddp_target.to(ddp_output.device)) + losses.append(loss) + + # Do multiple backwards and sync only at the last backward + for iteration in range(num_iterations): + loss = losses[iteration] + + if iteration < num_iterations - 1: + # Accumulate grads locally + accelerator.backward(loss) + + # DDP model and model should only be in sync after last backward + for param, ddp_param in zip(model.parameters(), ddp_model.parameters()): + if not param.requires_grad: + continue + # Grads should not be in sync + assert torch.allclose(param.grad, ddp_param.grad) is False, ( + f"Gradients in sync when they should not be:\nModel grad ({param.grad}) == DDP grad ({ddp_param.grad})" + ) + + else: + # Sync grads if last backward + with accelerator.trigger_sync_in_backward(ddp_model): + accelerator.backward(loss) + + # DDP model and model should only be in sync after last backward + for param, ddp_param in zip(model.parameters(), ddp_model.parameters()): + if not param.requires_grad: + continue + # Grads should be in sync + assert torch.allclose(param.grad, ddp_param.grad) is True, ( + f"Gradients not in sync when they should be:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})" + ) + + +def test_gradient_accumulation(split_batches=False, dispatch_batches=False, sync_each_batch=False): + gradient_accumulation_plugin = GradientAccumulationPlugin(num_steps=2, sync_each_batch=sync_each_batch) + dataloader_config = DataLoaderConfiguration(split_batches=split_batches, dispatch_batches=dispatch_batches) + accelerator = Accelerator( + dataloader_config=dataloader_config, + gradient_accumulation_plugin=gradient_accumulation_plugin, + ) + # Test that context manager behaves properly + model, ddp_model, dataloader = get_training_setup(accelerator) + for iteration, batch in enumerate(dataloader): + ddp_input, ddp_target = batch.values() + # Gather the distributed inputs and targs for the base model + input, target = accelerator.gather((ddp_input, ddp_target)) + input, target = input.to(accelerator.device), target.to(accelerator.device) + # Perform our initial ground truth step in non "DDP" + step_model(model, input, target, accelerator, False) + # Do "gradient accumulation" (noop) + with accelerator.accumulate(ddp_model): + step_model(ddp_model, ddp_input, ddp_target, accelerator) + + # DDP model and model should only be in sync when not (iteration % 2 == 0) + for param, ddp_param in zip(model.parameters(), ddp_model.parameters()): + if not param.requires_grad: + continue + if ((iteration + 1) % 2 == 0) or (iteration == len(dataloader) - 1) or sync_each_batch: + # Grads should be in sync + assert torch.allclose(param.grad, ddp_param.grad) is True, ( + f"Gradients not in sync when they should be at iteration {iteration}:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})" + ) + else: + # Grads should not be in sync + assert torch.allclose(param.grad, ddp_param.grad) is False, ( + f"Gradients in sync when they should not be at iteration {iteration}:\nModel grad ({param.grad}) == DDP grad ({ddp_param.grad})" + ) + + # Shuffle ddp_input on each iteration + torch.manual_seed(1337 + iteration) + ddp_input = ddp_input[torch.randperm(len(ddp_input))] + GradientState._reset_state() + + +def test_gradient_accumulation_with_opt_and_scheduler( + split_batches=False, dispatch_batches=False, sync_each_batch=False +): + gradient_accumulation_plugin = GradientAccumulationPlugin(num_steps=2, sync_each_batch=sync_each_batch) + dataloader_config = DataLoaderConfiguration(split_batches=split_batches, dispatch_batches=dispatch_batches) + accelerator = Accelerator( + dataloader_config=dataloader_config, + gradient_accumulation_plugin=gradient_accumulation_plugin, + ) + # Test that context manager behaves properly + model, opt, sched, dataloader, ddp_model, ddp_opt, ddp_sched = get_training_setup(accelerator, True) + for iteration, batch in enumerate(dataloader): + ddp_input, ddp_target = batch.values() + # Gather the distributed inputs and targs for the base model + input, target = accelerator.gather((ddp_input, ddp_target)) + input, target = input.to(accelerator.device), target.to(accelerator.device) + # Perform our initial ground truth step in non "DDP" + model.train() + ddp_model.train() + step_model(model, input, target, accelerator, False) + opt.step() + + if ((iteration + 1) % 2 == 0) or ((iteration + 1) == len(dataloader)): + if split_batches: + sched.step() + else: + for _ in range(accelerator.num_processes): + sched.step() + + # Perform gradient accumulation under wrapper + with accelerator.accumulate(ddp_model): + step_model(ddp_model, ddp_input, ddp_target, accelerator) + ddp_opt.step() + ddp_sched.step() + + # Learning rates should be the same + assert opt.param_groups[0]["lr"] == ddp_opt.param_groups[0]["lr"], ( + f"Learning rates found in each optimizer did not align\nopt: {opt.param_groups[0]['lr']}\nDDP opt: {ddp_opt.param_groups[0]['lr']}\n" + ) + did_step = (((iteration + 1) % 2) == 0) or ((iteration + 1) == len(dataloader)) + if accelerator.num_processes > 1: + check_model_parameters( + model, + ddp_model, + did_step or sync_each_batch, # syncs at each grad_accum interval of if sync_each_batch==True + iteration, + rtol=1e-2, # needs a relative tolerance due to roundoff errors + ) + + if did_step: + opt.zero_grad() # flush gradients every accum step + ddp_opt.zero_grad() + + # Shuffle ddp_input on each iteration + torch.manual_seed(1337 + iteration) + GradientState._reset_state() + + +def test_dataloader_break(): + accelerator = Accelerator() + first_dset = RegressionDataset(length=80) + first_dataloader = DataLoader(first_dset, batch_size=16) + second_dset = RegressionDataset(length=96) + second_dataloader = DataLoader(second_dset, batch_size=16) + first_dataloader, second_dataloader = accelerator.prepare(first_dataloader, second_dataloader) + + assert accelerator.gradient_state.active_dataloader is None + for iteration, _ in enumerate(first_dataloader): + assert id(accelerator.gradient_state.active_dataloader) == id(first_dataloader) + if iteration < len(first_dataloader) - 1: + assert not accelerator.gradient_state.end_of_dataloader + if iteration == 1: + for batch_num, _ in enumerate(second_dataloader): + assert id(accelerator.gradient_state.active_dataloader) == id(second_dataloader) + if batch_num < len(second_dataloader) - 1: + assert not accelerator.gradient_state.end_of_dataloader + else: + assert accelerator.gradient_state.end_of_dataloader + else: + assert accelerator.gradient_state.end_of_dataloader + assert accelerator.gradient_state.active_dataloader is None + + +def main(): + accelerator = Accelerator() + state = accelerator.state + if state.local_process_index == 0: + print("**Test `accumulate` gradient accumulation with dataloader break**") + if state.distributed_type != DistributedType.XLA: + test_dataloader_break() + if state.distributed_type == DistributedType.NO: + if state.local_process_index == 0: + print("**Test NOOP `no_sync` context manager**") + test_noop_sync(accelerator) + if state.distributed_type in ( + DistributedType.MULTI_GPU, + DistributedType.MULTI_NPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_CPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + ): + if state.local_process_index == 0: + print("**Test Distributed `no_sync` context manager**") + test_distributed_sync(accelerator) + if state.local_process_index == 0: + print("**Test Distributed `no_sync` context manager with multiple forwards**") + test_distributed_sync_multiple_fwd(accelerator) + if state.distributed_type in ( + DistributedType.MULTI_GPU, + DistributedType.MULTI_NPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + ): + for split_batch in [True, False]: + for dispatch_batches in [True, False]: + for sync_each_batch in [True, False]: + if state.local_process_index == 0: + print( + "**Test `accumulate` gradient accumulation, ", + f"`split_batches={split_batch}` and `dispatch_batches={dispatch_batches}` and `sync_each_batch={sync_each_batch}`**", + ) + test_gradient_accumulation(split_batch, dispatch_batches, sync_each_batch) + + # Currently will break on torch 2.0 +, need to investigate why + if state.local_process_index == 0: + print( + "**Test `accumulate` gradient accumulation with optimizer and scheduler, ", + "`split_batches=False`, `dispatch_batches=False`, `sync_each_batch=False`**", + ) + test_gradient_accumulation_with_opt_and_scheduler() + if state.distributed_type in ( + DistributedType.MULTI_GPU, + DistributedType.MULTI_NPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + ): + for split_batch in [True, False]: + for dispatch_batches in [True, False]: + for sync_each_batch in [True, False]: + if not split_batch and not dispatch_batches and not sync_each_batch: + continue + if state.local_process_index == 0: + print( + "**Test `accumulate` gradient accumulation with optimizer and scheduler, ", + f"`split_batches={split_batch}` and `dispatch_batches={dispatch_batches}` and `sync_each_batch={sync_each_batch}`**", + ) + test_gradient_accumulation_with_opt_and_scheduler(split_batch, dispatch_batches, sync_each_batch) + state.destroy_process_group() + + +def _mp_fn(index): + # For xla_spawn (TPUs) + main() + + +if __name__ == "__main__": + main() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/testing.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..c14f07b8d5518dd969cc22a87cf1c23480f1c06d --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/testing.py @@ -0,0 +1,896 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import inspect +import io +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from contextlib import contextmanager +from functools import partial +from pathlib import Path +from typing import Optional, Union +from unittest import mock + +import torch + +import accelerate + +from ..state import AcceleratorState +from ..utils import ( + check_cuda_fp8_capability, + compare_versions, + gather, + is_aim_available, + is_bnb_available, + is_clearml_available, + is_comet_ml_available, + is_cuda_available, + is_datasets_available, + is_deepspeed_available, + is_dvclive_available, + is_fp8_available, + is_fp16_available, + is_habana_gaudi1, + is_hpu_available, + is_import_timer_available, + is_matplotlib_available, + is_mlflow_available, + is_mlu_available, + is_mps_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_pandas_available, + is_pippy_available, + is_pytest_available, + is_schedulefree_available, + is_sdaa_available, + is_swanlab_available, + is_tensorboard_available, + is_timm_available, + is_torch_version, + is_torch_xla_available, + is_torchao_available, + is_torchdata_stateful_dataloader_available, + is_torchvision_available, + is_trackio_available, + is_transformer_engine_available, + is_transformer_engine_mxfp8_available, + is_transformers_available, + is_triton_available, + is_wandb_available, + is_xpu_available, + str_to_bool, +) + + +def get_backend(): + if is_torch_xla_available(): + return "xla", torch.cuda.device_count(), torch.cuda.memory_allocated + elif is_cuda_available(): + return "cuda", torch.cuda.device_count(), torch.cuda.memory_allocated + elif is_mps_available(min_version="2.0"): + return "mps", 1, torch.mps.current_allocated_memory + elif is_mps_available(): + return "mps", 1, lambda: 0 + elif is_mlu_available(): + return "mlu", torch.mlu.device_count(), torch.mlu.memory_allocated + elif is_sdaa_available(): + return "sdaa", torch.sdaa.device_count(), torch.sdaa.memory_allocated + elif is_musa_available(): + return "musa", torch.musa.device_count(), torch.musa.memory_allocated + elif is_npu_available(): + return "npu", torch.npu.device_count(), torch.npu.memory_allocated + elif is_xpu_available(): + return "xpu", torch.xpu.device_count(), torch.xpu.memory_allocated + elif is_hpu_available(): + return "hpu", torch.hpu.device_count(), torch.hpu.memory_allocated + elif is_neuron_available(): + return "neuron", torch.neuron.device_count(), torch.neuron.memory_allocated + else: + return "cpu", 1, lambda: 0 + + +torch_device, device_count, memory_allocated_func = get_backend() + + +def get_launch_command(**kwargs) -> list: + """ + Wraps around `kwargs` to help simplify launching from `subprocess`. + + Example: + ```python + # returns ['accelerate', 'launch', '--num_processes=2', '--device_count=2'] + get_launch_command(num_processes=2, device_count=2) + ``` + """ + command = ["accelerate", "launch"] + for k, v in kwargs.items(): + if isinstance(v, bool) and v: + command.append(f"--{k}") + elif v is not None: + command.append(f"--{k}={v}") + return command + + +DEFAULT_LAUNCH_COMMAND = get_launch_command(num_processes=device_count, monitor_interval=0.1) + + +def parse_flag_from_env(key, default=False): + try: + value = os.environ[key] + except KeyError: + # KEY isn't set, default to `default`. + _value = default + else: + # KEY is set, convert it to True or False. + try: + _value = str_to_bool(value) + except ValueError: + # More values are supported, but let's keep the message simple. + raise ValueError(f"If set, {key} must be yes or no.") + return _value + + +_run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False) + + +def skip(test_case): + "Decorator that skips a test unconditionally" + return unittest.skip("Test was skipped")(test_case) + + +def slow(test_case): + """ + Decorator marking a test as slow. Slow tests are skipped by default. Set the RUN_SLOW environment variable to a + truthy value to run them. + """ + return unittest.skipUnless(_run_slow_tests, "test is slow")(test_case) + + +def require_cpu(test_case): + """ + Decorator marking a test that must be only ran on the CPU. These tests are skipped when a GPU is available. + """ + return unittest.skipUnless(torch_device == "cpu", "test requires only a CPU")(test_case) + + +def require_non_cpu(test_case): + """ + Decorator marking a test that requires a hardware accelerator backend. These tests are skipped when there are no + hardware accelerator available. + """ + return unittest.skipUnless(torch_device != "cpu", "test requires a GPU")(test_case) + + +def require_cuda(test_case): + """ + Decorator marking a test that requires CUDA. These tests are skipped when there are no GPU available or when + TorchXLA is available. + """ + return unittest.skipUnless(is_cuda_available() and not is_torch_xla_available(), "test requires a GPU")(test_case) + + +def require_cuda_or_hpu(test_case): + """ + Decorator marking a test that requires CUDA or HPU. These tests are skipped when there are no GPU available or when + TorchXLA is available. + """ + return unittest.skipUnless( + (is_cuda_available() and not is_torch_xla_available()) or is_hpu_available(), "test requires a GPU or HPU" + )(test_case) + + +def require_xpu(test_case): + """ + Decorator marking a test that requires XPU. These tests are skipped when there are no XPU available. + """ + return unittest.skipUnless(is_xpu_available(), "test requires a XPU")(test_case) + + +def require_cuda_or_xpu(test_case): + """ + Decorator marking a test that requires CUDA or XPU. These tests are skipped when there are no GPU available or when + TorchXLA is available. + """ + cuda_condition = is_cuda_available() and not is_torch_xla_available() + xpu_condition = is_xpu_available() + return unittest.skipUnless(cuda_condition or xpu_condition, "test requires a CUDA GPU or XPU")(test_case) + + +def require_non_xpu(test_case): + """ + Decorator marking a test that should be skipped for XPU. + """ + return unittest.skipUnless(torch_device != "xpu", "test requires a non-XPU")(test_case) + + +def require_non_hpu(test_case): + """ + Decorator marking a test that should be skipped for HPU. + """ + return unittest.skipUnless(torch_device != "hpu", "test requires a non-HPU")(test_case) + + +def require_fp16(test_case): + """ + Decorator marking a test that requires FP16. These tests are skipped when FP16 is not supported. + """ + + return unittest.skipUnless(is_fp16_available(), "test requires FP16 support")(test_case) + + +def require_fp8(test_case): + """ + Decorator marking a test that requires FP8. These tests are skipped when FP8 is not supported. + """ + + # is_fp8_available only checks for libraries + # ideally it should check for device capability as well + fp8_is_available = is_fp8_available() + + if torch.cuda.is_available() and not check_cuda_fp8_capability(): + fp8_is_available = False + + if is_hpu_available() and is_habana_gaudi1(): + fp8_is_available = False + + return unittest.skipUnless(fp8_is_available, "test requires FP8 support")(test_case) + + +def require_fsdp2(test_case): + return unittest.skipUnless(is_torch_version(">=", "2.5.0"), "test requires FSDP2 (torch >= 2.5.0)")(test_case) + + +def require_mlu(test_case): + """ + Decorator marking a test that requires MLU. These tests are skipped when there are no MLU available. + """ + return unittest.skipUnless(is_mlu_available(), "test require a MLU")(test_case) + + +def require_sdaa(test_case): + """ + Decorator marking a test that requires SDAA. These tests are skipped when there are no SDAA available. + """ + return unittest.skipUnless(is_sdaa_available(), "test require a SDAA")(test_case) + + +def require_musa(test_case): + """ + Decorator marking a test that requires MUSA. These tests are skipped when there are no MUSA available. + """ + return unittest.skipUnless(is_musa_available(), "test require a MUSA")(test_case) + + +def require_npu(test_case): + """ + Decorator marking a test that requires NPU. These tests are skipped when there are no NPU available. + """ + return unittest.skipUnless(is_npu_available(), "test require a NPU")(test_case) + + +def require_neuron(test_case): + """ + Decorator marking a test that requires Neuron. These tests are skipped when there are no Neuron Cores available. + """ + return unittest.skipUnless(is_neuron_available(), "test require Neuron Cores")(test_case) + + +def require_mps(test_case): + """ + Decorator marking a test that requires MPS backend. These tests are skipped when torch doesn't support `mps` + backend. + """ + return unittest.skipUnless(is_mps_available(), "test requires a `mps` backend support in `torch`")(test_case) + + +def require_huggingface_suite(test_case): + """ + Decorator marking a test that requires transformers and datasets. These tests are skipped when they are not. + """ + return unittest.skipUnless( + is_transformers_available() and is_datasets_available(), + "test requires the Hugging Face suite", + )(test_case) + + +def require_datasets(test_case): + """ + Decorator marking a test that requires datasets. These tests are skipped when they are not. + """ + return unittest.skipUnless(is_datasets_available(), "test requires the datasets library")(test_case) + + +def require_transformers(test_case): + """ + Decorator marking a test that requires transformers. These tests are skipped when they are not. + """ + return unittest.skipUnless(is_transformers_available(), "test requires the transformers library")(test_case) + + +def require_timm(test_case): + """ + Decorator marking a test that requires timm. These tests are skipped when they are not. + """ + return unittest.skipUnless(is_timm_available(), "test requires the timm library")(test_case) + + +def require_torchvision(test_case): + """ + Decorator marking a test that requires torchvision. These tests are skipped when they are not. + """ + return unittest.skipUnless(is_torchvision_available(), "test requires the torchvision library")(test_case) + + +def require_triton(test_case): + """ + Decorator marking a test that requires triton. These tests are skipped when they are not. + """ + return unittest.skipUnless(is_triton_available(), "test requires the triton library")(test_case) + + +def require_schedulefree(test_case): + """ + Decorator marking a test that requires schedulefree. These tests are skipped when they are not. + """ + return unittest.skipUnless(is_schedulefree_available(), "test requires the schedulefree library")(test_case) + + +def require_bnb(test_case): + """ + Decorator marking a test that requires bitsandbytes. These tests are skipped when they are not. + """ + return unittest.skipUnless(is_bnb_available(), "test requires the bitsandbytes library")(test_case) + + +def require_tpu(test_case): + """ + Decorator marking a test that requires TPUs. These tests are skipped when there are no TPUs available. + """ + return unittest.skipUnless(is_torch_xla_available(check_is_tpu=True), "test requires TPU")(test_case) + + +def require_non_torch_xla(test_case): + """ + Decorator marking a test as requiring an environment without TorchXLA. These tests are skipped when TorchXLA is + available. + """ + return unittest.skipUnless(not is_torch_xla_available(), "test requires an env without TorchXLA")(test_case) + + +def require_single_device(test_case): + """ + Decorator marking a test that requires a single device. These tests are skipped when there is no hardware + accelerator available or number of devices is more than one. + """ + return unittest.skipUnless( + torch_device != "cpu" and device_count == 1, "test requires a single device accelerator" + )(test_case) + + +def require_single_gpu(test_case): + """ + Decorator marking a test that requires CUDA on a single GPU. These tests are skipped when there are no GPU + available or number of GPUs is more than one. + """ + return unittest.skipUnless(torch.cuda.device_count() == 1, "test requires a GPU")(test_case) + + +def require_single_xpu(test_case): + """ + Decorator marking a test that requires CUDA on a single XPU. These tests are skipped when there are no XPU + available or number of xPUs is more than one. + """ + return unittest.skipUnless(torch.xpu.device_count() == 1, "test requires a XPU")(test_case) + + +def require_multi_device(test_case): + """ + Decorator marking a test that requires a multi-device setup. These tests are skipped on a machine without multiple + devices. + """ + return unittest.skipUnless(device_count > 1, "test requires multiple hardware accelerators")(test_case) + + +def require_multi_gpu(test_case): + """ + Decorator marking a test that requires a multi-GPU setup. These tests are skipped on a machine without multiple + GPUs. + """ + return unittest.skipUnless(torch.cuda.device_count() > 1, "test requires multiple GPUs")(test_case) + + +def require_multi_xpu(test_case): + """ + Decorator marking a test that requires a multi-XPU setup. These tests are skipped on a machine without multiple + XPUs. + """ + return unittest.skipUnless(torch.xpu.device_count() > 1, "test requires multiple XPUs")(test_case) + + +def require_multi_gpu_or_xpu(test_case): + """ + Decorator marking a test that requires a multi-GPU setup. These tests are skipped on a machine without multiple + GPUs or XPUs. + """ + return unittest.skipUnless( + (is_cuda_available() or is_xpu_available()) and device_count > 1, "test requires multiple GPUs or XPUs" + )(test_case) + + +def require_deepspeed(test_case): + """ + Decorator marking a test that requires DeepSpeed installed. These tests are skipped when DeepSpeed isn't installed + """ + return unittest.skipUnless(is_deepspeed_available(), "test requires DeepSpeed")(test_case) + + +def require_tp(test_case): + """ + Decorator marking a test that requires TP installed. These tests are skipped when TP isn't installed + """ + return unittest.skipUnless( + is_torch_version(">=", "2.3.0") and compare_versions("transformers", ">=", "4.52.0"), + "test requires torch version >= 2.3.0 and transformers version >= 4.52.0", + )(test_case) + + +def require_torch_min_version(test_case=None, version=None): + """ + Decorator marking that a test requires a particular torch version to be tested. These tests are skipped when an + installed torch version is less than the required one. + """ + if test_case is None: + return partial(require_torch_min_version, version=version) + return unittest.skipUnless(is_torch_version(">=", version), f"test requires torch version >= {version}")(test_case) + + +def require_tensorboard(test_case): + """ + Decorator marking a test that requires tensorboard installed. These tests are skipped when tensorboard isn't + installed + """ + return unittest.skipUnless(is_tensorboard_available(), "test requires Tensorboard")(test_case) + + +def require_wandb(test_case): + """ + Decorator marking a test that requires wandb installed. These tests are skipped when wandb isn't installed + """ + return unittest.skipUnless(is_wandb_available(), "test requires wandb")(test_case) + + +def require_trackio(test_case): + """ + Decorator marking a test that requires trackio installed. These tests are skipped when trackio isn't installed + """ + return unittest.skipUnless(is_trackio_available(), "test requires trackio")(test_case) + + +def require_comet_ml(test_case): + """ + Decorator marking a test that requires comet_ml installed. These tests are skipped when comet_ml isn't installed + """ + return unittest.skipUnless(is_comet_ml_available(), "test requires comet_ml")(test_case) + + +def require_aim(test_case): + """ + Decorator marking a test that requires aim installed. These tests are skipped when aim isn't installed + """ + return unittest.skipUnless(is_aim_available(), "test requires aim")(test_case) + + +def require_clearml(test_case): + """ + Decorator marking a test that requires clearml installed. These tests are skipped when clearml isn't installed + """ + return unittest.skipUnless(is_clearml_available(), "test requires clearml")(test_case) + + +def require_dvclive(test_case): + """ + Decorator marking a test that requires dvclive installed. These tests are skipped when dvclive isn't installed + """ + return unittest.skipUnless(is_dvclive_available(), "test requires dvclive")(test_case) + + +def require_swanlab(test_case): + """ + Decorator marking a test that requires swanlab installed. These tests are skipped when swanlab isn't installed + """ + return unittest.skipUnless(is_swanlab_available(), "test requires swanlab")(test_case) + + +def require_pandas(test_case): + """ + Decorator marking a test that requires pandas installed. These tests are skipped when pandas isn't installed + """ + return unittest.skipUnless(is_pandas_available(), "test requires pandas")(test_case) + + +def require_mlflow(test_case): + """ + Decorator marking a test that requires mlflow installed. These tests are skipped when mlflow isn't installed + """ + return unittest.skipUnless(is_mlflow_available(), "test requires mlflow")(test_case) + + +def require_pippy(test_case): + """ + Decorator marking a test that requires pippy installed. These tests are skipped when pippy isn't installed It is + also checked if the test is running on a Gaudi1 device which doesn't support pippy. + """ + return unittest.skipUnless(is_pippy_available() and not is_habana_gaudi1(), "test requires pippy")(test_case) + + +def require_import_timer(test_case): + """ + Decorator marking a test that requires tuna interpreter installed. These tests are skipped when tuna isn't + installed + """ + return unittest.skipUnless(is_import_timer_available(), "test requires tuna interpreter")(test_case) + + +def require_transformer_engine(test_case): + """ + Decorator marking a test that requires transformers engine installed. These tests are skipped when transformers + engine isn't installed + """ + return unittest.skipUnless(is_transformer_engine_available(), "test requires transformers engine")(test_case) + + +def require_transformer_engine_mxfp8(test_case): + """ + Decorator marking a test that requires transformers engine MXFP8 block scaling available. These tests are skipped + when transformers engine MXFP8 block scaling isn't available + """ + return unittest.skipUnless( + is_transformer_engine_mxfp8_available(), "test requires transformers engine MXFP8 block scaling" + )(test_case) + + +def require_torchao(test_case): + """ + Decorator marking a test that requires torchao installed. These tests are skipped when torchao isn't installed + """ + return unittest.skipUnless(is_torchao_available(), "test requires torchao")(test_case) + + +def require_matplotlib(test_case): + """ + Decorator marking a test that requires matplotlib installed. These tests are skipped when matplotlib isn't + installed + """ + return unittest.skipUnless(is_matplotlib_available(), "test requires matplotlib")(test_case) + + +_atleast_one_tracker_available = ( + any([is_wandb_available(), is_tensorboard_available(), is_trackio_available(), is_swanlab_available()]) + and not is_comet_ml_available() +) + + +def require_trackers(test_case): + """ + Decorator marking that a test requires at least one tracking library installed. These tests are skipped when none + are installed + """ + return unittest.skipUnless( + _atleast_one_tracker_available, + "test requires at least one tracker to be available and for `comet_ml` to not be installed", + )(test_case) + + +def require_torchdata_stateful_dataloader(test_case): + """ + Decorator marking a test that requires torchdata.stateful_dataloader. + + These tests are skipped when torchdata with stateful_dataloader module isn't installed. + + """ + return unittest.skipUnless( + is_torchdata_stateful_dataloader_available(), "test requires torchdata.stateful_dataloader" + )(test_case) + + +def run_first(test_case): + """ + Decorator marking a test with order(1). When pytest-order plugin is installed, tests marked with this decorator are + guaranteed to run first. + + This is especially useful in some test settings like on a Gaudi instance where a Gaudi device can only be used by a + single process at a time. So we make sure all tests that run in a subprocess are launched first, to avoid device + allocation conflicts. + + If pytest is not installed, test will be returned as is. + """ + + if is_pytest_available(): + import pytest + + return pytest.mark.order(1)(test_case) + return test_case + + +class TempDirTestCase(unittest.TestCase): + """ + A TestCase class that keeps a single `tempfile.TemporaryDirectory` open for the duration of the class, wipes its + data at the start of a test, and then destroys it at the end of the TestCase. + + Useful for when a class or API requires a single constant folder throughout it's use, such as Weights and Biases + + The temporary directory location will be stored in `self.tmpdir` + """ + + clear_on_setup = True + + @classmethod + def setUpClass(cls): + "Creates a `tempfile.TemporaryDirectory` and stores it in `cls.tmpdir`" + cls.tmpdir = Path(tempfile.mkdtemp()) + + @classmethod + def tearDownClass(cls): + "Remove `cls.tmpdir` after test suite has finished" + if os.path.exists(cls.tmpdir): + shutil.rmtree(cls.tmpdir) + + def setUp(self): + "Destroy all contents in `self.tmpdir`, but not `self.tmpdir`" + if self.clear_on_setup: + for path in self.tmpdir.glob("**/*"): + if path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) + + +class AccelerateTestCase(unittest.TestCase): + """ + A TestCase class that will reset the accelerator state at the end of every test. Every test that checks or utilizes + the `AcceleratorState` class should inherit from this to avoid silent failures due to state being shared between + tests. + """ + + def tearDown(self): + super().tearDown() + # Reset the state of the AcceleratorState singleton. + AcceleratorState._reset_state(True) + + +class MockingTestCase(unittest.TestCase): + """ + A TestCase class designed to dynamically add various mockers that should be used in every test, mimicking the + behavior of a class-wide mock when defining one normally will not do. + + Useful when a mock requires specific information available only initialized after `TestCase.setUpClass`, such as + setting an environment variable with that information. + + The `add_mocks` function should be ran at the end of a `TestCase`'s `setUp` function, after a call to + `super().setUp()` such as: + ```python + def setUp(self): + super().setUp() + mocks = mock.patch.dict(os.environ, {"SOME_ENV_VAR", "SOME_VALUE"}) + self.add_mocks(mocks) + ``` + """ + + def add_mocks(self, mocks: Union[mock.Mock, list[mock.Mock]]): + """ + Add custom mocks for tests that should be repeated on each test. Should be called during + `MockingTestCase.setUp`, after `super().setUp()`. + + Args: + mocks (`mock.Mock` or list of `mock.Mock`): + Mocks that should be added to the `TestCase` after `TestCase.setUpClass` has been run + """ + self.mocks = mocks if isinstance(mocks, (tuple, list)) else [mocks] + for m in self.mocks: + m.start() + self.addCleanup(m.stop) + + +def are_the_same_tensors(tensor): + state = AcceleratorState() + tensor = tensor[None].clone().to(state.device) + tensors = gather(tensor).cpu() + tensor = tensor[0].cpu() + for i in range(tensors.shape[0]): + if not torch.equal(tensors[i], tensor): + return False + return True + + +class _RunOutput: + def __init__(self, returncode, stdout, stderr): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +async def _read_stream(stream, callback): + while True: + line = await stream.readline() + if line: + callback(line) + else: + break + + +async def _stream_subprocess(cmd, env=None, stdin=None, timeout=None, quiet=False, echo=False) -> _RunOutput: + if echo: + print("\nRunning: ", " ".join(cmd)) + + p = await asyncio.create_subprocess_exec( + cmd[0], + *cmd[1:], + stdin=stdin, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + + # note: there is a warning for a possible deadlock when using `wait` with huge amounts of data in the pipe + # https://docs.python.org/3/library/asyncio-subprocess.html#asyncio.asyncio.subprocess.Process.wait + # + # If it starts hanging, will need to switch to the following code. The problem is that no data + # will be seen until it's done and if it hangs for example there will be no debug info. + # out, err = await p.communicate() + # return _RunOutput(p.returncode, out, err) + + out = [] + err = [] + + def tee(line, sink, pipe, label=""): + line = line.decode("utf-8").rstrip() + sink.append(line) + if not quiet: + print(label, line, file=pipe) + + # XXX: the timeout doesn't seem to make any difference here + await asyncio.wait( + [ + asyncio.create_task(_read_stream(p.stdout, lambda l: tee(l, out, sys.stdout, label="stdout:"))), + asyncio.create_task(_read_stream(p.stderr, lambda l: tee(l, err, sys.stderr, label="stderr:"))), + ], + timeout=timeout, + ) + return _RunOutput(await p.wait(), out, err) + + +def execute_subprocess_async(cmd: list, env=None, stdin=None, timeout=180, quiet=False, echo=True) -> _RunOutput: + # Cast every path in `cmd` to a string + for i, c in enumerate(cmd): + if isinstance(c, Path): + cmd[i] = str(c) + + result = asyncio.run(_stream_subprocess(cmd, env=env, stdin=stdin, timeout=timeout, quiet=quiet, echo=echo)) + + cmd_str = " ".join(cmd) + if result.returncode > 0: + stderr = "\n".join(result.stderr) + raise RuntimeError( + f"'{cmd_str}' failed with returncode {result.returncode}\n\n" + f"The combined stderr from workers follows:\n{stderr}" + ) + + return result + + +def pytest_xdist_worker_id(): + """ + Returns an int value of worker's numerical id under `pytest-xdist`'s concurrent workers `pytest -n N` regime, or 0 + if `-n 1` or `pytest-xdist` isn't being used. + """ + worker = os.environ.get("PYTEST_XDIST_WORKER", "gw0") + worker = re.sub(r"^gw", "", worker, 0, re.M) + return int(worker) + + +def get_torch_dist_unique_port(): + """ + Returns a port number that can be fed to `torch.distributed.launch`'s `--master_port` argument. + + Under `pytest-xdist` it adds a delta number based on a worker id so that concurrent tests don't try to use the same + port at once. + """ + port = 29500 + uniq_delta = pytest_xdist_worker_id() + return port + uniq_delta + + +class SubprocessCallException(Exception): + pass + + +def run_command(command: list[str], return_stdout=False, env=None): + """ + Runs `command` with `subprocess.check_output` and will potentially return the `stdout`. Will also properly capture + if an error occurred while running `command` + """ + # Cast every path in `command` to a string + for i, c in enumerate(command): + if isinstance(c, Path): + command[i] = str(c) + if env is None: + env = os.environ.copy() + try: + output = subprocess.check_output(command, stderr=subprocess.STDOUT, env=env) + if return_stdout: + if hasattr(output, "decode"): + output = output.decode("utf-8") + return output + except subprocess.CalledProcessError as e: + raise SubprocessCallException( + f"Command `{' '.join(command)}` failed with the following error:\n\n{e.output.decode()}" + ) from e + + +def path_in_accelerate_package(*components: str) -> Path: + """ + Get a path within the `accelerate` package's directory. + + Args: + *components: Components of the path to join after the package directory. + + Returns: + `Path`: The path to the requested file or directory. + """ + + accelerate_package_dir = Path(inspect.getfile(accelerate)).parent + return accelerate_package_dir.joinpath(*components) + + +@contextmanager +def assert_exception(exception_class: Exception, msg: Optional[str] = None) -> bool: + """ + Context manager to assert that the right `Exception` class was raised. + + If `msg` is provided, will check that the message is contained in the raised exception. + """ + was_ran = False + try: + yield + was_ran = True + except Exception as e: + assert isinstance(e, exception_class), f"Expected exception of type {exception_class} but got {type(e)}" + if msg is not None: + assert msg in str(e), f"Expected message '{msg}' to be in exception but got '{str(e)}'" + if was_ran: + raise AssertionError(f"Expected exception of type {exception_class} but ran without issue.") + + +def capture_call_output(func, *args, **kwargs): + """ + Takes in a `func` with `args` and `kwargs` and returns the captured stdout as a string + """ + captured_output = io.StringIO() + original_stdout = sys.stdout + try: + sys.stdout = captured_output + func(*args, **kwargs) + except Exception as e: + raise e + finally: + sys.stdout = original_stdout + return captured_output.getvalue() diff --git a/kbidu_env/Lib/site-packages/accelerate/test_utils/training.py b/kbidu_env/Lib/site-packages/accelerate/test_utils/training.py new file mode 100644 index 0000000000000000000000000000000000000000..609912bf542aa4d7eccf475689db066a5a8897c4 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/test_utils/training.py @@ -0,0 +1,150 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch +from torch.utils.data import DataLoader + +from accelerate.utils.dataclasses import DistributedType + + +class RegressionDataset: + def __init__(self, a=2, b=3, length=64, seed=None): + rng = np.random.default_rng(seed) + self.length = length + self.x = rng.normal(size=(length,)).astype(np.float32) + self.y = a * self.x + b + rng.normal(scale=0.1, size=(length,)).astype(np.float32) + + def __len__(self): + return self.length + + def __getitem__(self, i): + return {"x": self.x[i], "y": self.y[i]} + + +class RegressionModel(torch.nn.Module): + def __init__(self, a=0, b=0, double_output=False): + super().__init__() + self.a = torch.nn.Parameter(torch.tensor(a).float()) + self.b = torch.nn.Parameter(torch.tensor(b).float()) + self.first_batch = True + + def forward(self, x=None): + if self.first_batch: + print(f"Model dtype: {self.a.dtype}, {self.b.dtype}. Input dtype: {x.dtype}") + self.first_batch = False + return x * self.a + self.b + + +def mocked_dataloaders(accelerator, batch_size: int = 16): + from datasets import load_dataset + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + data_files = {"train": "tests/test_samples/MRPC/train.csv", "validation": "tests/test_samples/MRPC/dev.csv"} + datasets = load_dataset("csv", data_files=data_files) + label_list = datasets["train"].unique("label") + + label_to_id = {v: i for i, v in enumerate(label_list)} + + def tokenize_function(examples): + # max_length=None => use the model max length (it's actually the default) + outputs = tokenizer( + examples["sentence1"], examples["sentence2"], truncation=True, max_length=None, padding="max_length" + ) + if "label" in examples: + outputs["labels"] = [label_to_id[l] for l in examples["label"]] + return outputs + + # Apply the method we just defined to all the examples in all the splits of the dataset + tokenized_datasets = datasets.map( + tokenize_function, + batched=True, + remove_columns=["sentence1", "sentence2", "label"], + ) + + def collate_fn(examples): + # On TPU it's best to pad everything to the same length or training will be very slow. + if accelerator.distributed_type == DistributedType.XLA: + return tokenizer.pad(examples, padding="max_length", max_length=128, return_tensors="pt") + return tokenizer.pad(examples, padding="longest", return_tensors="pt") + + # Instantiate dataloaders. + train_dataloader = DataLoader(tokenized_datasets["train"], shuffle=True, collate_fn=collate_fn, batch_size=2) + eval_dataloader = DataLoader(tokenized_datasets["validation"], shuffle=False, collate_fn=collate_fn, batch_size=1) + + return train_dataloader, eval_dataloader + + +def mocked_dataloaders_for_autoregressive_models(accelerator, batch_size: int = 16): + from datasets import load_dataset + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM-360M") + tokenizer.pad_token = tokenizer.eos_token + + data_files = {"train": "tests/test_samples/MRPC/train.csv", "validation": "tests/test_samples/MRPC/dev.csv"} + datasets = load_dataset("csv", data_files=data_files) + + def tokenize_function(examples): + # max_length=None => use the model max length (it's actually the default) + outputs = tokenizer(examples["sentence1"], truncation=True, max_length=None, return_attention_mask=False) + return outputs + + # Apply the method we just defined to all the examples in all the splits of the dataset + # starting with the main process first: + with accelerator.main_process_first(): + tokenized_datasets = datasets.map( + tokenize_function, + batched=True, + remove_columns=["sentence1", "sentence2", "label"], + ) + + def collate_fn(examples): + # On TPU it's best to pad everything to the same length or training will be very slow. + max_length = ( + 128 + if accelerator.distributed_type == DistributedType.XLA + else max([len(e["input_ids"]) for e in examples]) + ) + # When using mixed precision we want round multiples of 8/16 + if accelerator.mixed_precision == "fp8": + pad_to_multiple_of = 16 + elif accelerator.mixed_precision != "no": + pad_to_multiple_of = 8 + else: + pad_to_multiple_of = None + + batch = tokenizer.pad( + examples, + padding="max_length", + max_length=max_length + 1, + pad_to_multiple_of=pad_to_multiple_of, + return_tensors="pt", + ) + + batch["labels"] = batch["input_ids"][:, 1:] + batch["input_ids"] = batch["input_ids"][:, :-1] + if "attention_mask" in batch: + batch["attention_mask"] = batch["attention_mask"][:, :-1] + + batch["labels"] = torch.where(batch["labels"] == tokenizer.pad_token_id, -100, batch["labels"]) + + return batch + + # Instantiate dataloaders. + train_dataloader = DataLoader(tokenized_datasets["train"], shuffle=False, collate_fn=collate_fn, batch_size=2) + eval_dataloader = DataLoader(tokenized_datasets["validation"], shuffle=False, collate_fn=collate_fn, batch_size=1) + + return train_dataloader, eval_dataloader diff --git a/kbidu_env/Lib/site-packages/accelerate/tracking.py b/kbidu_env/Lib/site-packages/accelerate/tracking.py new file mode 100644 index 0000000000000000000000000000000000000000..3f42629216c9788f5780d28a161f64b7cc66ac7b --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/tracking.py @@ -0,0 +1,1315 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Expectation: +# Provide a project dir name, then each type of logger gets stored in project/{`logging_dir`} + +import json +import os +import time +from functools import wraps +from typing import Any, Optional, Union + +import yaml +from packaging import version + +from .logging import get_logger +from .state import PartialState +from .utils import ( + LoggerType, + compare_versions, + is_aim_available, + is_clearml_available, + is_comet_ml_available, + is_dvclive_available, + is_mlflow_available, + is_swanlab_available, + is_tensorboard_available, + is_trackio_available, + is_wandb_available, + listify, +) + + +_available_trackers = [] + +if is_tensorboard_available(): + _available_trackers.append(LoggerType.TENSORBOARD) + +if is_wandb_available(): + _available_trackers.append(LoggerType.WANDB) + +if is_comet_ml_available(): + _available_trackers.append(LoggerType.COMETML) + +if is_aim_available(): + _available_trackers.append(LoggerType.AIM) + +if is_mlflow_available(): + _available_trackers.append(LoggerType.MLFLOW) + +if is_clearml_available(): + _available_trackers.append(LoggerType.CLEARML) + +if is_dvclive_available(): + _available_trackers.append(LoggerType.DVCLIVE) + +if is_swanlab_available(): + _available_trackers.append(LoggerType.SWANLAB) + +if is_trackio_available(): + _available_trackers.append(LoggerType.TRACKIO) + +logger = get_logger(__name__) + + +def on_main_process(function): + """ + Decorator to selectively run the decorated function on the main process only based on the `main_process_only` + attribute in a class. + + Checks at function execution rather than initialization time, not triggering the initialization of the + `PartialState`. + """ + + @wraps(function) + def execute_on_main_process(self, *args, **kwargs): + if getattr(self, "main_process_only", False): + return PartialState().on_main_process(function)(self, *args, **kwargs) + else: + return function(self, *args, **kwargs) + + return execute_on_main_process + + +def get_available_trackers(): + "Returns a list of all supported available trackers in the system" + return _available_trackers + + +class GeneralTracker: + """ + A base Tracker class to be used for all logging integration implementations. + + Each function should take in `**kwargs` that will automatically be passed in from a base dictionary provided to + [`Accelerator`]. + + Should implement `name`, `requires_logging_directory`, and `tracker` properties such that: + + `name` (`str`): String representation of the tracker class name, such as "TensorBoard" `requires_logging_directory` + (`bool`): Whether the logger requires a directory to store their logs. `tracker` (`object`): Should return internal + tracking mechanism used by a tracker class (such as the `run` for wandb) + + Implementations can also include a `main_process_only` (`bool`) attribute to toggle if relevant logging, init, and + other functions should occur on the main process or across all processes (by default will use `True`) + """ + + main_process_only = True + + def __init__(self, _blank=False): + if not _blank: + err = "" + if not hasattr(self, "name"): + err += "`name`" + if not hasattr(self, "requires_logging_directory"): + if len(err) > 0: + err += ", " + err += "`requires_logging_directory`" + + # as tracker is a @property that relies on post-init + if "tracker" not in dir(self): + if len(err) > 0: + err += ", " + err += "`tracker`" + if len(err) > 0: + raise NotImplementedError( + f"The implementation for this tracker class is missing the following " + f"required attributes. Please define them in the class definition: " + f"{err}" + ) + + def start(self): + """ + Lazy initialization of the tracker inside Accelerator to avoid initializing PartialState before + InitProcessGroupKwargs. + """ + + def store_init_configuration(self, values: dict): + """ + Logs `values` as hyperparameters for the run. Implementations should use the experiment configuration + functionality of a tracking API. + + Args: + values (Dictionary `str` to `bool`, `str`, `float` or `int`): + Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`, + `str`, `float`, `int`, or `None`. + """ + + def log(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `values` to the current run. Base `log` implementations of a tracking API should go in here, along with + special behavior for the `step parameter. + + Args: + values (Dictionary `str` to `str`, `float`, or `int`): + Values to be logged as key-value pairs. The values need to have type `str`, `float`, or `int`. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + """ + + def finish(self): + """ + Should run any finalizing functions within the tracking API. If the API should not have one, just don't + overwrite that method. + """ + + +class TensorBoardTracker(GeneralTracker): + """ + A `Tracker` class that supports `tensorboard`. Should be initialized at the start of your script. + + Args: + run_name (`str`): + The name of the experiment run + logging_dir (`str`, `os.PathLike`): + Location for TensorBoard logs to be stored. + **kwargs (additional keyword arguments, *optional*): + Additional key word arguments passed along to the `tensorboard.SummaryWriter.__init__` method. + """ + + name = "tensorboard" + requires_logging_directory = True + + def __init__(self, run_name: str, logging_dir: Union[str, os.PathLike], **kwargs): + super().__init__() + self.run_name = run_name + self.logging_dir_param = logging_dir + self.init_kwargs = kwargs + + @on_main_process + def start(self): + try: + from torch.utils import tensorboard + except ModuleNotFoundError: + import tensorboardX as tensorboard + self.logging_dir = os.path.join(self.logging_dir_param, self.run_name) + self.writer = tensorboard.SummaryWriter(self.logging_dir, **self.init_kwargs) + logger.debug(f"Initialized TensorBoard project {self.run_name} logging to {self.logging_dir}") + logger.debug( + "Make sure to log any initial configurations with `self.store_init_configuration` before training!" + ) + + @property + def tracker(self): + return self.writer + + @on_main_process + def store_init_configuration(self, values: dict): + """ + Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. Stores the + hyperparameters in a yaml file for future use. + + Args: + values (Dictionary `str` to `bool`, `str`, `float` or `int`): + Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`, + `str`, `float`, `int`, or `None`. + """ + self.writer.add_hparams(values, metric_dict={}) + self.writer.flush() + project_run_name = time.time() + dir_name = os.path.join(self.logging_dir, str(project_run_name)) + os.makedirs(dir_name, exist_ok=True) + with open(os.path.join(dir_name, "hparams.yml"), "w") as outfile: + try: + yaml.dump(values, outfile) + except yaml.representer.RepresenterError: + logger.error("Serialization to store hyperparameters failed") + raise + logger.debug("Stored initial configuration hyperparameters to TensorBoard and hparams yaml file") + + @on_main_process + def log(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `values` to the current run. + + Args: + values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`): + Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of + `str` to `float`/`int`. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to either `SummaryWriter.add_scaler`, + `SummaryWriter.add_text`, or `SummaryWriter.add_scalers` method based on the contents of `values`. + """ + values = listify(values) + for k, v in values.items(): + if isinstance(v, (int, float)): + self.writer.add_scalar(k, v, global_step=step, **kwargs) + elif isinstance(v, str): + self.writer.add_text(k, v, global_step=step, **kwargs) + elif isinstance(v, dict): + self.writer.add_scalars(k, v, global_step=step, **kwargs) + self.writer.flush() + logger.debug("Successfully logged to TensorBoard") + + @on_main_process + def log_images(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `images` to the current run. + + Args: + values (Dictionary `str` to `List` of `np.ndarray` or `PIL.Image`): + Values to be logged as key-value pairs. The values need to have type `List` of `np.ndarray` or + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to the `SummaryWriter.add_image` method. + """ + for k, v in values.items(): + self.writer.add_images(k, v, global_step=step, **kwargs) + logger.debug("Successfully logged images to TensorBoard") + + @on_main_process + def finish(self): + """ + Closes `TensorBoard` writer + """ + self.writer.close() + logger.debug("TensorBoard writer closed") + + +class WandBTracker(GeneralTracker): + """ + A `Tracker` class that supports `wandb`. Should be initialized at the start of your script. + + Args: + run_name (`str`): + The name of the experiment run. + **kwargs (additional keyword arguments, *optional*): + Additional key word arguments passed along to the `wandb.init` method. + """ + + name = "wandb" + requires_logging_directory = False + main_process_only = False + + def __init__(self, run_name: str, **kwargs): + super().__init__() + self.run_name = run_name + self.init_kwargs = kwargs + + @on_main_process + def start(self): + import wandb + + self.run = wandb.init(project=self.run_name, **self.init_kwargs) + logger.debug(f"Initialized WandB project {self.run_name}") + logger.debug( + "Make sure to log any initial configurations with `self.store_init_configuration` before training!" + ) + + @property + def tracker(self): + return self.run + + @on_main_process + def store_init_configuration(self, values: dict): + """ + Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. + + Args: + values (Dictionary `str` to `bool`, `str`, `float` or `int`): + Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`, + `str`, `float`, `int`, or `None`. + """ + import wandb + + wandb.config.update(values, allow_val_change=True) + logger.debug("Stored initial configuration hyperparameters to WandB") + + @on_main_process + def log(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `values` to the current run. + + Args: + values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`): + Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of + `str` to `float`/`int`. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to the `wandb.log` method. + """ + self.run.log(values, step=step, **kwargs) + logger.debug("Successfully logged to WandB") + + @on_main_process + def log_images(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `images` to the current run. + + Args: + values (Dictionary `str` to `List` of `np.ndarray` or `PIL.Image`): + Values to be logged as key-value pairs. The values need to have type `List` of `np.ndarray` or + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to the `wandb.log` method. + """ + import wandb + + for k, v in values.items(): + self.log({k: [wandb.Image(image) for image in v]}, step=step, **kwargs) + logger.debug("Successfully logged images to WandB") + + @on_main_process + def log_table( + self, + table_name: str, + columns: Optional[list[str]] = None, + data: Optional[list[list[Any]]] = None, + dataframe: Any = None, + step: Optional[int] = None, + **kwargs, + ): + """ + Log a Table containing any object type (text, image, audio, video, molecule, html, etc). Can be defined either + with `columns` and `data` or with `dataframe`. + + Args: + table_name (`str`): + The name to give to the logged table on the wandb workspace + columns (list of `str`, *optional*): + The name of the columns on the table + data (List of List of Any data type, *optional*): + The data to be logged in the table + dataframe (Any data type, *optional*): + The data to be logged in the table + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + """ + import wandb + + values = {table_name: wandb.Table(columns=columns, data=data, dataframe=dataframe)} + self.log(values, step=step, **kwargs) + + @on_main_process + def finish(self): + """ + Closes `wandb` writer + """ + self.run.finish() + logger.debug("WandB run closed") + + +class TrackioTracker(GeneralTracker): + """ + A `Tracker` class that supports `trackio`. Should be initialized at the start of your script. + + Args: + run_name (`str`): + The name of the experiment run. Will be used as the `project` name when instantiating trackio. + **kwargs (additional keyword arguments, *optional*): + Additional key word arguments passed along to the `trackio.init` method. Refer to this + [init](https://github.com/gradio-app/trackio/blob/814809552310468b13f84f33764f1369b4e5136c/trackio/__init__.py#L22) + to see all supported key word arguments. + """ + + name = "trackio" + requires_logging_directory = False + main_process_only = False + + def __init__(self, run_name: str, **kwargs): + super().__init__() + self.run_name = run_name + self.init_kwargs = kwargs + + @on_main_process + def start(self): + import trackio + + self.run = trackio.init(project=self.run_name, **self.init_kwargs) + logger.debug(f"Initialized trackio project {self.run_name}") + logger.debug( + "Make sure to log any initial configurations with `self.store_init_configuration` before training!" + ) + + @property + def tracker(self): + return self.run + + @on_main_process + def store_init_configuration(self, values: dict): + """ + Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. + + Args: + values (Dictionary `str` to `bool`, `str`, `float` or `int`): + Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`, + `str`, `float`, `int`, or `None`. + """ + import trackio + + trackio.config.update(values, allow_val_change=True) + logger.debug("Stored initial configuration hyperparameters to trackio") + + @on_main_process + def log(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `values` to the current run. + + Args: + values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`): + Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of + `str` to `float`/`int`. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to the `trackio.log` method. + """ + self.run.log(values, step=step, **kwargs) + logger.debug("Successfully logged to trackio") + + @on_main_process + def finish(self): + """ + Closes `trackio` run + """ + self.run.finish() + logger.debug("trackio run closed") + + +class CometMLTracker(GeneralTracker): + """ + A `Tracker` class that supports `comet_ml`. Should be initialized at the start of your script. + + API keys must be stored in a Comet config file. + + Note: + For `comet_ml` versions < 3.41.0, additional keyword arguments are passed to `comet_ml.Experiment` instead: + https://www.comet.com/docs/v2/api-and-sdk/python-sdk/reference/Experiment/#comet_ml.Experiment.__init__ + + Args: + run_name (`str`): + The name of the experiment run. + **kwargs (additional keyword arguments, *optional*): + Additional key word arguments passed along to the `comet_ml.start` method: + https://www.comet.com/docs/v2/api-and-sdk/python-sdk/reference/start/ + """ + + name = "comet_ml" + requires_logging_directory = False + + def __init__(self, run_name: str, **kwargs): + super().__init__() + self.run_name = run_name + self.init_kwargs = kwargs + + @on_main_process + def start(self): + import comet_ml + + comet_version = version.parse(comet_ml.__version__) + if compare_versions(comet_version, ">=", "3.41.0"): + self.writer = comet_ml.start(project_name=self.run_name, **self.init_kwargs) + else: + logger.info("Update `comet_ml` (>=3.41.0) for experiment reuse and offline support.") + self.writer = comet_ml.Experiment(project_name=self.run_name, **self.init_kwargs) + + logger.debug(f"Initialized CometML project {self.run_name}") + logger.debug( + "Make sure to log any initial configurations with `self.store_init_configuration` before training!" + ) + + @property + def tracker(self): + return self.writer + + @on_main_process + def store_init_configuration(self, values: dict): + """ + Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. + + Args: + values (Dictionary `str` to `bool`, `str`, `float` or `int`): + Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`, + `str`, `float`, `int`, or `None`. + """ + self.writer.log_parameters(values) + logger.debug("Stored initial configuration hyperparameters to Comet") + + @on_main_process + def log(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `values` to the current run. + + Args: + values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`): + Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of + `str` to `float`/`int`. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to either `Experiment.log_metric`, `Experiment.log_other`, + or `Experiment.log_metrics` method based on the contents of `values`. + """ + if step is not None: + self.writer.set_step(step) + for k, v in values.items(): + if isinstance(v, (int, float)): + self.writer.log_metric(k, v, step=step, **kwargs) + elif isinstance(v, str): + self.writer.log_other(k, v, **kwargs) + elif isinstance(v, dict): + self.writer.log_metrics(v, step=step, **kwargs) + logger.debug("Successfully logged to Comet") + + @on_main_process + def finish(self): + """ + Flush `comet-ml` writer + """ + self.writer.end() + logger.debug("Comet run flushed") + + +class AimTracker(GeneralTracker): + """ + A `Tracker` class that supports `aim`. Should be initialized at the start of your script. + + Args: + run_name (`str`): + The name of the experiment run. + **kwargs (additional keyword arguments, *optional*): + Additional key word arguments passed along to the `Run.__init__` method. + """ + + name = "aim" + requires_logging_directory = True + + def __init__(self, run_name: str, logging_dir: Optional[Union[str, os.PathLike]] = ".", **kwargs): + super().__init__() + self.run_name = run_name + self.aim_repo_path = logging_dir + self.init_kwargs = kwargs + + @on_main_process + def start(self): + from aim import Run + + self.writer = Run(repo=self.aim_repo_path, **self.init_kwargs) + self.writer.name = self.run_name + logger.debug(f"Initialized Aim project {self.run_name}") + logger.debug( + "Make sure to log any initial configurations with `self.store_init_configuration` before training!" + ) + + @property + def tracker(self): + return self.writer + + @on_main_process + def store_init_configuration(self, values: dict): + """ + Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. + + Args: + values (`dict`): + Values to be stored as initial hyperparameters as key-value pairs. + """ + self.writer["hparams"] = values + + @on_main_process + def log(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `values` to the current run. + + Args: + values (`dict`): + Values to be logged as key-value pairs. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to the `Run.track` method. + """ + # Note: replace this with the dictionary support when merged + for key, value in values.items(): + self.writer.track(value, name=key, step=step, **kwargs) + + @on_main_process + def log_images(self, values: dict, step: Optional[int] = None, kwargs: Optional[dict[str, dict]] = None): + """ + Logs `images` to the current run. + + Args: + values (`Dict[str, Union[np.ndarray, PIL.Image, Tuple[np.ndarray, str], Tuple[PIL.Image, str]]]`): + Values to be logged as key-value pairs. The values need to have type `np.ndarray` or PIL.Image. If a + tuple is provided, the first element should be the image and the second element should be the caption. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs (`Dict[str, dict]`): + Additional key word arguments passed along to the `Run.Image` and `Run.track` method specified by the + keys `aim_image` and `track`, respectively. + """ + import aim + + aim_image_kw = {} + track_kw = {} + + if kwargs is not None: + aim_image_kw = kwargs.get("aim_image", {}) + track_kw = kwargs.get("track", {}) + + for key, value in values.items(): + if isinstance(value, tuple): + img, caption = value + else: + img, caption = value, "" + aim_image = aim.Image(img, caption=caption, **aim_image_kw) + self.writer.track(aim_image, name=key, step=step, **track_kw) + + @on_main_process + def finish(self): + """ + Closes `aim` writer + """ + self.writer.close() + + +class MLflowTracker(GeneralTracker): + """ + A `Tracker` class that supports `mlflow`. Should be initialized at the start of your script. + + Args: + experiment_name (`str`, *optional*): + Name of the experiment. Environment variable MLFLOW_EXPERIMENT_NAME has priority over this argument. + logging_dir (`str` or `os.PathLike`, defaults to `"."`): + Location for mlflow logs to be stored. + run_id (`str`, *optional*): + If specified, get the run with the specified UUID and log parameters and metrics under that run. The run’s + end time is unset and its status is set to running, but the run’s other attributes (source_version, + source_type, etc.) are not changed. Environment variable MLFLOW_RUN_ID has priority over this argument. + tags (`Dict[str, str]`, *optional*): + An optional `dict` of `str` keys and values, or a `str` dump from a `dict`, to set as tags on the run. If a + run is being resumed, these tags are set on the resumed run. If a new run is being created, these tags are + set on the new run. Environment variable MLFLOW_TAGS has priority over this argument. + nested_run (`bool`, *optional*, defaults to `False`): + Controls whether run is nested in parent run. True creates a nested run. Environment variable + MLFLOW_NESTED_RUN has priority over this argument. + run_name (`str`, *optional*): + Name of new run (stored as a mlflow.runName tag). Used only when `run_id` is unspecified. + description (`str`, *optional*): + An optional string that populates the description box of the run. If a run is being resumed, the + description is set on the resumed run. If a new run is being created, the description is set on the new + run. + """ + + name = "mlflow" + requires_logging_directory = False + + def __init__( + self, + experiment_name: Optional[str] = None, + logging_dir: Optional[Union[str, os.PathLike]] = None, + run_id: Optional[str] = None, + tags: Optional[Union[dict[str, Any], str]] = None, + nested_run: Optional[bool] = False, + run_name: Optional[str] = None, + description: Optional[str] = None, + ): + experiment_name = os.environ.get("MLFLOW_EXPERIMENT_NAME", experiment_name) + run_id = os.environ.get("MLFLOW_RUN_ID", run_id) + tags = os.environ.get("MLFLOW_TAGS", tags) + if isinstance(tags, str): + tags = json.loads(tags) + + nested_run = os.environ.get("MLFLOW_NESTED_RUN", nested_run) + + self.experiment_name = experiment_name + self.logging_dir = logging_dir + self.run_id = run_id + self.tags = tags + self.nested_run = nested_run + self.run_name = run_name + self.description = description + + @on_main_process + def start(self): + import mlflow + + exps = mlflow.search_experiments(filter_string=f"name = '{self.experiment_name}'") + if len(exps) > 0: + if len(exps) > 1: + logger.warning("Multiple experiments with the same name found. Using first one.") + experiment_id = exps[0].experiment_id + else: + experiment_id = mlflow.create_experiment( + name=self.experiment_name, + artifact_location=self.logging_dir, + tags=self.tags, + ) + + self.active_run = mlflow.start_run( + run_id=self.run_id, + experiment_id=experiment_id, + run_name=self.run_name, + nested=self.nested_run, + tags=self.tags, + description=self.description, + ) + + logger.debug(f"Initialized mlflow experiment {self.experiment_name}") + logger.debug( + "Make sure to log any initial configurations with `self.store_init_configuration` before training!" + ) + + @property + def tracker(self): + return self.active_run + + @on_main_process + def store_init_configuration(self, values: dict): + """ + Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. + + Args: + values (`dict`): + Values to be stored as initial hyperparameters as key-value pairs. + """ + import mlflow + + values_list = [] + for name, value in values.items(): + # internally, all values are converted to str in MLflow + if len(str(value)) > mlflow.utils.validation.MAX_PARAM_VAL_LENGTH: + logger.warning_once( + f'Accelerate is attempting to log a value of "{value}" for key "{name}" as a parameter. MLflow\'s' + f" log_param() only accepts values no longer than {mlflow.utils.validation.MAX_PARAM_VAL_LENGTH} characters so we dropped this attribute." + ) + else: + values_list.append((name, value)) + + # MLflow cannot log more than 100 values in one go, so we have to split it + for i in range(0, len(values_list), mlflow.utils.validation.MAX_PARAMS_TAGS_PER_BATCH): + mlflow.log_params(dict(values_list[i : i + mlflow.utils.validation.MAX_PARAMS_TAGS_PER_BATCH])) + + logger.debug("Stored initial configuration hyperparameters to MLflow") + + @on_main_process + def log(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `values` to the current run. + + Args: + values (`dict`): + Values to be logged as key-value pairs. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to the `mlflow.log_metrics` method. + """ + metrics = {} + for k, v in values.items(): + if isinstance(v, (int, float)): + metrics[k] = v + else: + logger.warning_once( + f'MLflowTracker is attempting to log a value of "{v}" of type {type(v)} for key "{k}" as a metric. ' + "MLflow's log_metric() only accepts float and int types so we dropped this attribute." + ) + import mlflow + + mlflow.log_metrics(metrics, step=step, **kwargs) + logger.debug("Successfully logged to mlflow") + + @on_main_process + def log_figure(self, figure: Any, artifact_file: str, **save_kwargs): + """ + Logs an figure to the current run. + + Args: + figure (Any): + The figure to be logged. + artifact_file (`str`, *optional*): + The run-relative artifact file path in posixpath format to which the image is saved. + If not provided, the image is saved to a default location. + **kwargs: + Additional keyword arguments passed to the underlying mlflow.log_image function. + """ + import mlflow + + mlflow.log_figure(figure=figure, artifact_file=artifact_file, **save_kwargs) + logger.debug("Successfully logged image to mlflow") + + @on_main_process + def log_artifacts(self, local_dir: str, artifact_path: Optional[str] = None): + """ + Logs an artifacts (all content of a dir) to the current run. + + local_dir (`str`): + Path to the directory to be logged as an artifact. + artifact_path (`str`, *optional*): + Directory within the run's artifact directory where the artifact will be logged. If omitted, the + artifact will be logged to the root of the run's artifact directory. The run step. If included, the + artifact will be affiliated with this step. + """ + import mlflow + + mlflow.log_artifacts(local_dir=local_dir, artifact_path=artifact_path) + logger.debug("Successfully logged artofact to mlflow") + + @on_main_process + def log_artifact(self, local_path: str, artifact_path: Optional[str] = None): + """ + Logs an artifact (file) to the current run. + + local_path (`str`): + Path to the file to be logged as an artifact. + artifact_path (`str`, *optional*): + Directory within the run's artifact directory where the artifact will be logged. If omitted, the + artifact will be logged to the root of the run's artifact directory. The run step. If included, the + artifact will be affiliated with this step. + """ + import mlflow + + mlflow.log_artifact(local_path=local_path, artifact_path=artifact_path) + logger.debug("Successfully logged artofact to mlflow") + + @on_main_process + def finish(self): + """ + End the active MLflow run. + """ + import mlflow + + mlflow.end_run() + + +class ClearMLTracker(GeneralTracker): + """ + A `Tracker` class that supports `clearml`. Should be initialized at the start of your script. + + Args: + run_name (`str`, *optional*): + Name of the experiment. Environment variables `CLEARML_PROJECT` and `CLEARML_TASK` have priority over this + argument. + **kwargs (additional keyword arguments, *optional*): + Kwargs passed along to the `Task.__init__` method. + """ + + name = "clearml" + requires_logging_directory = False + + def __init__(self, run_name: Optional[str] = None, **kwargs): + super().__init__() + self.user_provided_run_name = run_name + self._initialized_externally = False + self.init_kwargs = kwargs + + @on_main_process + def start(self): + from clearml import Task + + current_task = Task.current_task() + if current_task: + self._initialized_externally = True + self.task = current_task + return + + task_init_args = {**self.init_kwargs} + task_init_args.setdefault("project_name", os.environ.get("CLEARML_PROJECT", self.user_provided_run_name)) + task_init_args.setdefault("task_name", os.environ.get("CLEARML_TASK", self.user_provided_run_name)) + self.task = Task.init(**task_init_args) + + @property + def tracker(self): + return self.task + + @on_main_process + def store_init_configuration(self, values: dict): + """ + Connect configuration dictionary to the Task object. Should be run at the beginning of your experiment. + + Args: + values (`dict`): + Values to be stored as initial hyperparameters as key-value pairs. + """ + return self.task.connect_configuration(values) + + @on_main_process + def log(self, values: dict[str, Union[int, float]], step: Optional[int] = None, **kwargs): + """ + Logs `values` dictionary to the current run. The dictionary keys must be strings. The dictionary values must be + ints or floats + + Args: + values (`Dict[str, Union[int, float]]`): + Values to be logged as key-value pairs. If the key starts with 'eval_'/'test_'/'train_', the value will + be reported under the 'eval'/'test'/'train' series and the respective prefix will be removed. + Otherwise, the value will be reported under the 'train' series, and no prefix will be removed. + step (`int`, *optional*): + If specified, the values will be reported as scalars, with the iteration number equal to `step`. + Otherwise they will be reported as single values. + kwargs: + Additional key word arguments passed along to the `clearml.Logger.report_single_value` or + `clearml.Logger.report_scalar` methods. + """ + clearml_logger = self.task.get_logger() + for k, v in values.items(): + if not isinstance(v, (int, float)): + logger.warning_once( + "Accelerator is attempting to log a value of " + f'"{v}" of type {type(v)} for key "{k}" as a scalar. ' + "This invocation of ClearML logger's report_scalar() " + "is incorrect so we dropped this attribute." + ) + continue + if step is None: + clearml_logger.report_single_value(name=k, value=v, **kwargs) + continue + title, series = ClearMLTracker._get_title_series(k) + clearml_logger.report_scalar(title=title, series=series, value=v, iteration=step, **kwargs) + + @on_main_process + def log_images(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `images` to the current run. + + Args: + values (`Dict[str, List[Union[np.ndarray, PIL.Image]]`): + Values to be logged as key-value pairs. The values need to have type `List` of `np.ndarray` or + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to the `clearml.Logger.report_image` method. + """ + clearml_logger = self.task.get_logger() + for k, v in values.items(): + title, series = ClearMLTracker._get_title_series(k) + clearml_logger.report_image(title=title, series=series, iteration=step, image=v, **kwargs) + + @on_main_process + def log_table( + self, + table_name: str, + columns: Optional[list[str]] = None, + data: Optional[list[list[Any]]] = None, + dataframe: Any = None, + step: Optional[int] = None, + **kwargs, + ): + """ + Log a Table to the task. Can be defined eitherwith `columns` and `data` or with `dataframe`. + + Args: + table_name (`str`): + The name of the table + columns (list of `str`, *optional*): + The name of the columns on the table + data (List of List of Any data type, *optional*): + The data to be logged in the table. If `columns` is not specified, then the first entry in data will be + the name of the columns of the table + dataframe (Any data type, *optional*): + The data to be logged in the table + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to the `clearml.Logger.report_table` method. + """ + to_report = dataframe + if dataframe is None: + if data is None: + raise ValueError( + "`ClearMLTracker.log_table` requires that `data` to be supplied if `dataframe` is `None`" + ) + to_report = [columns] + data if columns else data + title, series = ClearMLTracker._get_title_series(table_name) + self.task.get_logger().report_table(title=title, series=series, table_plot=to_report, iteration=step, **kwargs) + + @on_main_process + def finish(self): + """ + Close the ClearML task. If the task was initialized externally (e.g. by manually calling `Task.init`), this + function is a noop + """ + if self.task and not self._initialized_externally: + self.task.close() + + @staticmethod + def _get_title_series(name): + for prefix in ["eval", "test", "train"]: + if name.startswith(prefix + "_"): + return name[len(prefix) + 1 :], prefix + return name, "train" + + +class DVCLiveTracker(GeneralTracker): + """ + A `Tracker` class that supports `dvclive`. Should be initialized at the start of your script. + + Args: + run_name (`str`, *optional*): + Ignored for dvclive. See `kwargs` instead. + kwargs: + Additional key word arguments passed along to [`dvclive.Live()`](https://dvc.org/doc/dvclive/live). + + Example: + + ```py + from accelerate import Accelerator + + accelerator = Accelerator(log_with="dvclive") + accelerator.init_trackers(project_name="my_project", init_kwargs={"dvclive": {"dir": "my_directory"}}) + ``` + """ + + name = "dvclive" + requires_logging_directory = False + + def __init__(self, run_name: Optional[str] = None, live: Optional[Any] = None, **kwargs): + super().__init__() + self.live = live + self.init_kwargs = kwargs + + @on_main_process + def start(self): + from dvclive import Live + + self.live = self.live if self.live is not None else Live(**self.init_kwargs) + + @property + def tracker(self): + return self.live + + @on_main_process + def store_init_configuration(self, values: dict): + """ + Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. Stores the + hyperparameters in a yaml file for future use. + + Args: + values (Dictionary `str` to `bool`, `str`, `float`, `int`, or a List or Dict of those types): + Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`, + `str`, `float`, or `int`. + """ + self.live.log_params(values) + + @on_main_process + def log(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `values` to the current run. + + Args: + values (Dictionary `str` to `str`, `float`, or `int`): + Values to be logged as key-value pairs. The values need to have type `str`, `float`, or `int`. + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to `dvclive.Live.log_metric()`. + """ + from dvclive.plots import Metric + + if step is not None: + self.live.step = step + for k, v in values.items(): + if Metric.could_log(v): + self.live.log_metric(k, v, **kwargs) + else: + logger.warning_once( + "Accelerator attempted to log a value of " + f'"{v}" of type {type(v)} for key "{k}" as a scalar. ' + "This invocation of DVCLive's Live.log_metric() " + "is incorrect so we dropped this attribute." + ) + self.live.next_step() + + @on_main_process + def finish(self): + """ + Closes `dvclive.Live()`. + """ + self.live.end() + + +class SwanLabTracker(GeneralTracker): + """ + A `Tracker` class that supports `swanlab`. Should be initialized at the start of your script. + + Args: + run_name (`str`): + The name of the experiment run. + **kwargs (additional keyword arguments, *optional*): + Additional key word arguments passed along to the `swanlab.init` method. + """ + + name = "swanlab" + requires_logging_directory = False + main_process_only = False + + def __init__(self, run_name: str, **kwargs): + super().__init__() + self.run_name = run_name + self.init_kwargs = kwargs + + @on_main_process + def start(self): + import swanlab + + self.run = swanlab.init(project=self.run_name, **self.init_kwargs) + swanlab.config["FRAMEWORK"] = "🤗Accelerate" # add accelerate logo in config + logger.debug(f"Initialized SwanLab project {self.run_name}") + logger.debug( + "Make sure to log any initial configurations with `self.store_init_configuration` before training!" + ) + + @property + def tracker(self): + return self.run + + @on_main_process + def store_init_configuration(self, values: dict): + """ + Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. + + Args: + values (Dictionary `str` to `bool`, `str`, `float` or `int`): + Values to be stored as initial hyperparameters as key-value pairs. The values need to have type `bool`, + `str`, `float`, `int`, or `None`. + """ + import swanlab + + swanlab.config.update(values, allow_val_change=True) + logger.debug("Stored initial configuration hyperparameters to SwanLab") + + @on_main_process + def log(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `values` to the current run. + + Args: + data : Dict[str, DataType] + Data must be a dict. The key must be a string with 0-9, a-z, A-Z, " ", "_", "-", "/". The value must be a + `float`, `float convertible object`, `int` or `swanlab.data.BaseType`. + step : int, optional + The step number of the current data, if not provided, it will be automatically incremented. + If step is duplicated, the data will be ignored. + kwargs: + Additional key word arguments passed along to the `swanlab.log` method. Likes: + print_to_console : bool, optional + Whether to print the data to the console, the default is False. + """ + self.run.log(values, step=step, **kwargs) + logger.debug("Successfully logged to SwanLab") + + @on_main_process + def log_images(self, values: dict, step: Optional[int] = None, **kwargs): + """ + Logs `images` to the current run. + + Args: + values (Dictionary `str` to `List` of `np.ndarray` or `PIL.Image`): + Values to be logged as key-value pairs. The values need to have type `List` of `np.ndarray` or + step (`int`, *optional*): + The run step. If included, the log will be affiliated with this step. + kwargs: + Additional key word arguments passed along to the `swanlab.log` method. Likes: + print_to_console : bool, optional + Whether to print the data to the console, the default is False. + """ + import swanlab + + for k, v in values.items(): + self.log({k: [swanlab.Image(image) for image in v]}, step=step, **kwargs) + logger.debug("Successfully logged images to SwanLab") + + @on_main_process + def finish(self): + """ + Closes `swanlab` writer + """ + self.run.finish() + logger.debug("SwanLab run closed") + + +LOGGER_TYPE_TO_CLASS = { + "aim": AimTracker, + "comet_ml": CometMLTracker, + "mlflow": MLflowTracker, + "tensorboard": TensorBoardTracker, + "wandb": WandBTracker, + "clearml": ClearMLTracker, + "dvclive": DVCLiveTracker, + "swanlab": SwanLabTracker, + "trackio": TrackioTracker, +} + + +def filter_trackers( + log_with: list[Union[str, LoggerType, GeneralTracker]], + logging_dir: Optional[Union[str, os.PathLike]] = None, +): + """ + Takes in a list of potential tracker types and checks that: + - The tracker wanted is available in that environment + - Filters out repeats of tracker types + - If `all` is in `log_with`, will return all trackers in the environment + - If a tracker requires a `logging_dir`, ensures that `logging_dir` is not `None` + + Args: + log_with (list of `str`, [`~utils.LoggerType`] or [`~tracking.GeneralTracker`], *optional*): + A list of loggers to be setup for experiment tracking. Should be one or several of: + + - `"all"` + - `"tensorboard"` + - `"wandb"` + - `"trackio"` + - `"aim"` + - `"comet_ml"` + - `"mlflow"` + - `"dvclive"` + - `"swanlab"` + If `"all"` is selected, will pick up all available trackers in the environment and initialize them. Can + also accept implementations of `GeneralTracker` for custom trackers, and can be combined with `"all"`. + logging_dir (`str`, `os.PathLike`, *optional*): + A path to a directory for storing logs of locally-compatible loggers. + """ + loggers = [] + if log_with is not None: + if not isinstance(log_with, (list, tuple)): + log_with = [log_with] + if "all" in log_with or LoggerType.ALL in log_with: + loggers = [o for o in log_with if issubclass(type(o), GeneralTracker)] + get_available_trackers() + else: + for log_type in log_with: + if log_type not in LoggerType and not issubclass(type(log_type), GeneralTracker): + raise ValueError(f"Unsupported logging capability: {log_type}. Choose between {LoggerType.list()}") + if issubclass(type(log_type), GeneralTracker): + loggers.append(log_type) + else: + log_type = LoggerType(log_type) + if log_type not in loggers: + if log_type in get_available_trackers(): + tracker_init = LOGGER_TYPE_TO_CLASS[str(log_type)] + if tracker_init.requires_logging_directory: + if logging_dir is None: + raise ValueError( + f"Logging with `{log_type}` requires a `logging_dir` to be passed in." + ) + loggers.append(log_type) + else: + logger.debug(f"Tried adding logger {log_type}, but package is unavailable in the system.") + + return loggers diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__init__.py b/kbidu_env/Lib/site-packages/accelerate/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d24daf28dc547610678df0a67d20ea9c9bdb450c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/__init__.py @@ -0,0 +1,307 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..parallelism_config import ParallelismConfig +from .ao import convert_model_to_fp8_ao, filter_first_and_last_linear_layers, has_ao_layers +from .constants import ( + MITA_PROFILING_AVAILABLE_PYTORCH_VERSION, + MODEL_NAME, + OPTIMIZER_NAME, + PROFILE_PATTERN_NAME, + RNG_STATE_NAME, + SAFE_MODEL_NAME, + SAFE_WEIGHTS_INDEX_NAME, + SAFE_WEIGHTS_NAME, + SAFE_WEIGHTS_PATTERN_NAME, + SAMPLER_NAME, + SCALER_NAME, + SCHEDULER_NAME, + TORCH_DISTRIBUTED_OPERATION_TYPES, + TORCH_LAUNCH_PARAMS, + WEIGHTS_INDEX_NAME, + WEIGHTS_NAME, + WEIGHTS_PATTERN_NAME, + XPU_PROFILING_AVAILABLE_PYTORCH_VERSION, +) +from .dataclasses import ( + AORecipeKwargs, + AutocastKwargs, + BnbQuantizationConfig, + ComputeEnvironment, + CustomDtype, + DataLoaderConfiguration, + DDPCommunicationHookType, + DeepSpeedPlugin, + DeepSpeedSequenceParallelConfig, + DistributedDataParallelKwargs, + DistributedType, + DynamoBackend, + FP8RecipeKwargs, + FullyShardedDataParallelPlugin, + GradientAccumulationPlugin, + GradScalerKwargs, + InitProcessGroupKwargs, + KwargsHandler, + LoggerType, + MegatronLMPlugin, + MSAMPRecipeKwargs, + PrecisionType, + ProfileKwargs, + ProjectConfiguration, + RNGType, + SageMakerDistributedType, + TensorInformation, + TERecipeKwargs, + TorchContextParallelConfig, + TorchDynamoPlugin, + TorchTensorParallelConfig, + TorchTensorParallelPlugin, + add_model_config_to_megatron_parser, +) +from .environment import ( + are_libraries_initialized, + check_cuda_fp8_capability, + check_cuda_p2p_ib_support, + clear_environment, + convert_dict_to_env_variables, + get_cpu_distributed_information, + get_current_device_type, + get_gpu_info, + get_int_from_env, + parse_choice_from_env, + parse_flag_from_env, + patch_environment, + purge_accelerate_environment, + set_numa_affinity, + str_to_bool, +) +from .imports import ( + deepspeed_required, + is_4bit_bnb_available, + is_8bit_bnb_available, + is_aim_available, + is_amdsmi_available, + is_bf16_available, + is_bitsandbytes_multi_backend_available, + is_bnb_available, + is_boto3_available, + is_clearml_available, + is_comet_ml_available, + is_cuda_available, + is_datasets_available, + is_deepspeed_available, + is_dvclive_available, + is_fp8_available, + is_fp16_available, + is_habana_gaudi1, + is_hpu_available, + is_import_timer_available, + is_lomo_available, + is_matplotlib_available, + is_megatron_lm_available, + is_mlflow_available, + is_mlu_available, + is_mps_available, + is_msamp_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_pandas_available, + is_peft_available, + is_pippy_available, + is_pynvml_available, + is_pytest_available, + is_rich_available, + is_rocm_available, + is_sagemaker_available, + is_schedulefree_available, + is_sdaa_available, + is_swanlab_available, + is_tensorboard_available, + is_timm_available, + is_torch_xla_available, + is_torchao_available, + is_torchdata_available, + is_torchdata_stateful_dataloader_available, + is_torchvision_available, + is_trackio_available, + is_transformer_engine_available, + is_transformer_engine_mxfp8_available, + is_transformers_available, + is_triton_available, + is_wandb_available, + is_weights_only_available, + is_xccl_available, + is_xpu_available, + torchao_required, +) +from .modeling import ( + align_module_device, + calculate_maximum_sizes, + check_device_map, + check_tied_parameters_in_config, + check_tied_parameters_on_same_device, + compute_module_sizes, + convert_file_size_to_int, + dtype_byte_size, + find_tied_parameters, + get_balanced_memory, + get_grad_scaler, + get_max_layer_size, + get_max_memory, + get_mixed_precision_context_manager, + has_offloaded_params, + id_tensor_storage, + infer_auto_device_map, + is_peft_model, + load_checkpoint_in_model, + load_offloaded_weights, + load_state_dict, + named_module_tensors, + retie_parameters, + set_module_tensor_to_device, +) +from .offload import ( + OffloadedWeightsLoader, + PrefixedDataset, + extract_submodules_state_dict, + load_offloaded_weight, + offload_state_dict, + offload_weight, + save_offload_index, +) +from .operations import ( + CannotPadNestedTensorWarning, + GatheredParameters, + broadcast, + broadcast_object_list, + concatenate, + convert_outputs_to_fp32, + convert_to_fp32, + copy_tensor_to_devices, + find_batch_size, + find_device, + gather, + gather_object, + get_data_structure, + honor_type, + ignorant_find_batch_size, + initialize_tensors, + is_namedtuple, + is_tensor_information, + is_torch_tensor, + listify, + pad_across_processes, + pad_input_tensors, + recursively_apply, + reduce, + send_to_device, + slice_tensors, +) +from .versions import compare_versions, is_torch_version + + +if is_deepspeed_available(): + from .deepspeed import ( + DeepSpeedEngineWrapper, + DeepSpeedOptimizerWrapper, + DeepSpeedSchedulerWrapper, + DummyOptim, + DummyScheduler, + HfDeepSpeedConfig, + get_active_deepspeed_plugin, + map_pytorch_optim_to_deepspeed, + ) + +from .bnb import has_4bit_bnb_layers, load_and_quantize_model +from .fsdp_utils import ( + disable_fsdp_ram_efficient_loading, + enable_fsdp_ram_efficient_loading, + ensure_weights_retied, + fsdp2_apply_ac, + fsdp2_canonicalize_names, + fsdp2_load_full_state_dict, + fsdp2_prepare_model, + fsdp2_switch_optimizer_parameters, + get_fsdp2_grad_scaler, + load_fsdp_model, + load_fsdp_optimizer, + merge_fsdp_weights, + save_fsdp_model, + save_fsdp_optimizer, +) +from .launch import ( + PrepareForLaunch, + _filter_args, + prepare_deepspeed_cmd_env, + prepare_multi_gpu_env, + prepare_sagemager_args_inputs, + prepare_simple_launcher_cmd_env, + prepare_tpu, +) + +# For docs +from .megatron_lm import ( + AbstractTrainStep, + BertTrainStep, + GPTTrainStep, + MegatronLMDummyDataLoader, + MegatronLMDummyScheduler, + T5TrainStep, + avg_losses_across_data_parallel_group, +) + + +if is_megatron_lm_available(): + from .megatron_lm import ( + MegatronEngine, + MegatronLMOptimizerWrapper, + MegatronLMSchedulerWrapper, + gather_across_data_parallel_groups, + ) + from .megatron_lm import initialize as megatron_lm_initialize + from .megatron_lm import prepare_data_loader as megatron_lm_prepare_data_loader + from .megatron_lm import prepare_model_optimizer_scheduler as megatron_lm_prepare_model_optimizer_scheduler + from .megatron_lm import prepare_optimizer as megatron_lm_prepare_optimizer + from .megatron_lm import prepare_scheduler as megatron_lm_prepare_scheduler +from .memory import find_executable_batch_size, release_memory +from .other import ( + check_os_kernel, + clean_state_dict_for_safetensors, + compile_regions, + compile_regions_deepspeed, + compile_regions_fsdp2, + convert_bytes, + extract_model_from_parallel, + get_module_children_bottom_up, + get_pretty_name, + has_compiled_regions, + is_compiled_module, + is_port_in_use, + load, + merge_dicts, + model_has_dtensor, + recursive_getattr, + save, + wait_for_everyone, + write_basic_config, +) +from .random import set_seed, synchronize_rng_state, synchronize_rng_states +from .torch_xla import install_xla +from .tqdm import tqdm +from .transformer_engine import ( + apply_fp8_autowrap, + contextual_fp8_autocast, + convert_model, + has_transformer_engine_layers, +) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..58b6c466fae75d4b262a51daeb7257bf5388b142 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/ao.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/ao.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b0a2f9b88e8d0ad661bafb5011b6d4a6ffd30bec Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/ao.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/bnb.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/bnb.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb8edc123cbe32060c5b01d052ca6d347bb5b134 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/bnb.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/constants.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/constants.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1a9a26d8c9c3c7c9cca43cfd804fcee848091e4 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/constants.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/dataclasses.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/dataclasses.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf93a8e5889eab8aaf008b3f02f8afdca43c658b --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/dataclasses.cpython-314.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c51e216a19ab0ea67d7dd7e93a6e1c9a5971d5c21f03e8d0cbf143f64314d2a +size 175315 diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/deepspeed.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/deepspeed.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e295a1d1a0508ef77c393b86b8c7fc32fc04483 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/deepspeed.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/environment.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/environment.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..11e737e978e0982577629329f4d19fce718324b5 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/environment.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/fsdp_utils.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/fsdp_utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..40ac4ab6fe311ed6865e973d0b7a3ae7c3f8c608 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/fsdp_utils.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/imports.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/imports.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ef735a9ed5191b15422e74fbbbe03cb5bc6b5e1 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/imports.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/launch.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/launch.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..696565b993b6e131860321fe1e02f1804e68d0df Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/launch.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/megatron_lm.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/megatron_lm.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38fc42deeb3ef0b18331b088d4682ebc0dd1e41c Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/megatron_lm.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/memory.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/memory.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cce1860b2161cfcdd9d7a7f5e907974ca087a490 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/memory.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/modeling.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/modeling.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f50767b421a02741e33cb03a3fe4139cf84877d --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/modeling.cpython-314.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0a260a6f4b3ec16af87809cc3026c1a748f37cfe2fa15b8656b75951a98fe9c +size 109682 diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/offload.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/offload.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a2a191586e3e702a643a615f7ad2daa6de5d0df Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/offload.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/operations.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/operations.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4927906de63dc5e3a2c22dd459c03a8d22ff5701 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/operations.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/other.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/other.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..86dd5f343539dce063ebb69989f07f84f1f317a7 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/other.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/random.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/random.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53f890ca38f7da656354f0393084194b3b48e9f2 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/random.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/rich.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/rich.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8122442146da0d18ccd7c7a51c6e20402d655fb Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/rich.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/torch_xla.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/torch_xla.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..16210bf549c58ddbd2aad9f31671b4a31c2a53b5 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/torch_xla.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/tqdm.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/tqdm.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66218c9cdc62948f1e6a0e2ac011208c90cca5d1 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/tqdm.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/transformer_engine.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/transformer_engine.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f35da9f6aefee867b5255bf5341ce5c269f83e05 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/transformer_engine.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/versions.cpython-314.pyc b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/versions.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b5b41fcb69f644aba3f91e1282d17a2fe366286 Binary files /dev/null and b/kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/versions.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/ao.py b/kbidu_env/Lib/site-packages/accelerate/utils/ao.py new file mode 100644 index 0000000000000000000000000000000000000000..5cc417d72829006da38549b42fd68caae27b1204 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/ao.py @@ -0,0 +1,143 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Needed utilities for torchao FP8 training. +""" + +from functools import partial +from typing import TYPE_CHECKING, Callable, Optional + +import torch + +from .imports import is_torchao_available, torchao_required + + +if TYPE_CHECKING: + if is_torchao_available(): + from torchao.float8.float8_linear import Float8LinearConfig + + +def find_first_last_linear_layers(model: torch.nn.Module): + """ + Finds the first and last linear layer names in a model. + + This is needed during FP8 to avoid issues with instability by keeping the first and last layers unquantized. + + Ref: https://x.com/xariusrke/status/1826669142604141052 + """ + first_linear, last_linear = None, None + for name, module in model.named_modules(): + if isinstance(module, torch.nn.Linear): + if first_linear is None: + first_linear = name + last_linear = name + return first_linear, last_linear + + +def filter_linear_layers(module, fqn: str, layers_to_filter: list[str]) -> bool: + """ + A function which will check if `module` is: + - a `torch.nn.Linear` layer + - has in_features and out_features divisible by 16 + - is not part of `layers_to_filter` + + Args: + module (`torch.nn.Module`): + The module to check. + fqn (`str`): + The fully qualified name of the layer. + layers_to_filter (`List[str]`): + The list of layers to filter. + """ + if isinstance(module, torch.nn.Linear): + if module.in_features % 16 != 0 or module.out_features % 16 != 0: + return False + if fqn in layers_to_filter: + return False + return True + + +def filter_first_and_last_linear_layers(module, fqn: str) -> bool: + """ + A filter function which will filter out all linear layers except the first and last. + + + + For stability reasons, we skip the first and last linear layers Otherwise can lead to the model not training or + converging properly + + + + Args: + module (`torch.nn.Module`): + The module to check. + fqn (`str`): + The fully qualified name of the layer. + """ + first_linear, last_linear = find_first_last_linear_layers(module) + return filter_linear_layers(module, fqn, layers_to_filter=[first_linear, last_linear]) + + +@torchao_required +def has_ao_layers(model: torch.nn.Module): + from torchao.float8.float8_linear import Float8Linear + + for name, module in model.named_modules(): + if isinstance(module, Float8Linear): + return True + return False + + +@torchao_required +def convert_model_to_fp8_ao( + model: torch.nn.Module, + config: Optional["Float8LinearConfig"] = None, + module_filter_func: Optional[Callable] = filter_first_and_last_linear_layers, +): + """ + Converts all `nn.Linear` layers in the model (except the first and last) to torchao's `Float8Linear` layer inplace. + + Args: + model (`torch.nn.Module`): + The model to convert. + config (`torchao.float8.Float8LinearConfig`, *optional*): + The configuration for the FP8 training. Recommended to utilize + `torchao.float8.recipe_name_to_linear_config` to generate this. In general, the default config should be + sufficient (what is passed when set to `None`). + module_filter_func (`Callable`, *optional*, defaults to `filter_linear_layers`): + Optional function that must take in a module and layer name, and returns a boolean indicating whether the + module should be converted to FP8. Defaults to `filter_linear_layers`. See it for an example. + + Example: + + ```python + from accelerate.utils.ao import convert_model_to_fp8_ao + from accelerate import Accelerator + + accelerator = Accelerator( + + model = MyModel() + model.to(accelerator.device) + convert_to_float8_training(model) + + model.train() + ``` + """ + from torchao.float8 import convert_to_float8_training + + first_linear, last_linear = find_first_last_linear_layers(model) + if module_filter_func is None: + module_filter_func = partial(filter_linear_layers, layers_to_filter=[first_linear, last_linear]) + convert_to_float8_training(model, module_filter_fn=module_filter_func, config=config) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/bnb.py b/kbidu_env/Lib/site-packages/accelerate/utils/bnb.py new file mode 100644 index 0000000000000000000000000000000000000000..187b88e22caf55626c4f02767d91720a0c0e730e --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/bnb.py @@ -0,0 +1,473 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +from copy import deepcopy +from typing import Optional, Union + +import torch +import torch.nn as nn + +from accelerate.utils.imports import ( + is_4bit_bnb_available, + is_8bit_bnb_available, +) + +from ..big_modeling import dispatch_model, init_empty_weights +from .dataclasses import BnbQuantizationConfig +from .modeling import ( + find_tied_parameters, + get_balanced_memory, + infer_auto_device_map, + load_checkpoint_in_model, + offload_weight, + set_module_tensor_to_device, +) + + +logger = logging.getLogger(__name__) + + +def load_and_quantize_model( + model: torch.nn.Module, + bnb_quantization_config: BnbQuantizationConfig, + weights_location: Optional[Union[str, os.PathLike]] = None, + device_map: Optional[dict[str, Union[int, str, torch.device]]] = None, + no_split_module_classes: Optional[list[str]] = None, + max_memory: Optional[dict[Union[int, str], Union[int, str]]] = None, + offload_folder: Optional[Union[str, os.PathLike]] = None, + offload_state_dict: bool = False, +): + """ + This function will quantize the input model with the associated config passed in `bnb_quantization_config`. If the + model is in the meta device, we will load and dispatch the weights according to the `device_map` passed. If the + model is already loaded, we will quantize the model and put the model on the GPU, + + Args: + model (`torch.nn.Module`): + Input model. The model can be already loaded or on the meta device + bnb_quantization_config (`BnbQuantizationConfig`): + The bitsandbytes quantization parameters + weights_location (`str` or `os.PathLike`): + The folder weights_location to load. It can be: + - a path to a file containing a whole model state dict + - a path to a `.json` file containing the index to a sharded checkpoint + - a path to a folder containing a unique `.index.json` file and the shards of a checkpoint. + - a path to a folder containing a unique pytorch_model.bin file. + device_map (`Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer + name, once a given module name is inside, every submodule of it will be sent to the same device. + no_split_module_classes (`List[str]`, *optional*): + A list of layer class names that should never be split across device (for instance any layer that has a + residual connection). + max_memory (`Dict`, *optional*): + A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + If the `device_map` contains any value `"disk"`, the folder where we will offload weights. + offload_state_dict (`bool`, *optional*, defaults to `False`): + If `True`, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if + the weight of the CPU state dict + the biggest shard does not fit. + + Returns: + `torch.nn.Module`: The quantized model + """ + + load_in_4bit = bnb_quantization_config.load_in_4bit + load_in_8bit = bnb_quantization_config.load_in_8bit + + if load_in_8bit and not is_8bit_bnb_available(): + raise ImportError( + "You have a version of `bitsandbytes` that is not compatible with 8bit quantization," + " make sure you have the latest version of `bitsandbytes` installed." + ) + if load_in_4bit and not is_4bit_bnb_available(): + raise ValueError( + "You have a version of `bitsandbytes` that is not compatible with 4bit quantization," + "make sure you have the latest version of `bitsandbytes` installed." + ) + + modules_on_cpu = [] + # custom device map + if isinstance(device_map, dict) and len(device_map.keys()) > 1: + modules_on_cpu = [key for key, value in device_map.items() if value in ["disk", "cpu"]] + + # We keep some modules such as the lm_head in their original dtype for numerical stability reasons + if bnb_quantization_config.skip_modules is None: + bnb_quantization_config.skip_modules = get_keys_to_not_convert(model) + + # add cpu modules to skip modules only for 4-bit modules + if load_in_4bit: + bnb_quantization_config.skip_modules.extend(modules_on_cpu) + modules_to_not_convert = bnb_quantization_config.skip_modules + + # We add the modules we want to keep in full precision + if bnb_quantization_config.keep_in_fp32_modules is None: + bnb_quantization_config.keep_in_fp32_modules = [] + keep_in_fp32_modules = bnb_quantization_config.keep_in_fp32_modules + modules_to_not_convert.extend(keep_in_fp32_modules) + + # compatibility with peft + model.is_loaded_in_4bit = load_in_4bit + model.is_loaded_in_8bit = load_in_8bit + + model_device = get_parameter_device(model) + if model_device.type != "meta": + # quantization of an already loaded model + logger.warning( + "It is not recommended to quantize a loaded model. " + "The model should be instantiated under the `init_empty_weights` context manager." + ) + model = replace_with_bnb_layers(model, bnb_quantization_config, modules_to_not_convert=modules_to_not_convert) + # convert param to the right dtype + # remove_duplicate=False so tied params (e.g. BLOOM's lm_head.weight tied to + # word_embeddings.weight) are visited under every alias — the keep_in_fp32 cast + # would otherwise be skipped if the tied alias came first under a different name. + dtype = bnb_quantization_config.torch_dtype + for name, param in model.named_parameters(remove_duplicate=False): + if any(module_to_keep_in_fp32 in name for module_to_keep_in_fp32 in keep_in_fp32_modules): + param.data = param.data.to(torch.float32) + elif torch.is_floating_point(param): + param.data = param.data.to(dtype) + # Second pass: ensure keep_in_fp32 modules are in fp32 even when their weights are tied + # to other modules (named_parameters() deduplicates tied params, so the first pass may miss them) + for name, module in model.named_modules(): + if any(module_to_keep_in_fp32 in name for module_to_keep_in_fp32 in keep_in_fp32_modules): + for param in module.parameters(recurse=False): + param.data = param.data.to(torch.float32) + if model_device.type == "cuda": + model.cuda(torch.cuda.current_device()) + torch.cuda.empty_cache() + elif torch.cuda.is_available(): + model.to(torch.cuda.current_device()) + elif torch.xpu.is_available(): + model.to(torch.xpu.current_device()) + else: + raise RuntimeError("No GPU or Intel XPU found. A GPU or Intel XPU is needed for quantization.") + logger.info( + f"The model device type is {model_device.type}. However, gpu or intel xpu is needed for quantization." + "We move the model to it." + ) + return model + + elif weights_location is None: + raise RuntimeError( + f"`weights_location` needs to be the folder path containing the weights of the model, but we found {weights_location} " + ) + + else: + with init_empty_weights(): + model = replace_with_bnb_layers( + model, bnb_quantization_config, modules_to_not_convert=modules_to_not_convert + ) + device_map = get_quantized_model_device_map( + model, + bnb_quantization_config, + device_map, + max_memory=max_memory, + no_split_module_classes=no_split_module_classes, + ) + if offload_state_dict is None and device_map is not None and "disk" in device_map.values(): + offload_state_dict = True + + offload = any(x in list(device_map.values()) for x in ["cpu", "disk"]) + + load_checkpoint_in_model( + model, + weights_location, + device_map, + dtype=bnb_quantization_config.torch_dtype, + offload_folder=offload_folder, + offload_state_dict=offload_state_dict, + keep_in_fp32_modules=bnb_quantization_config.keep_in_fp32_modules, + offload_8bit_bnb=load_in_8bit and offload, + ) + return dispatch_model(model, device_map=device_map, offload_dir=offload_folder) + + +def get_quantized_model_device_map( + model, bnb_quantization_config, device_map=None, max_memory=None, no_split_module_classes=None +): + if device_map is None: + if torch.cuda.is_available(): + device_map = {"": torch.cuda.current_device()} + elif torch.xpu.is_available(): + device_map = {"": torch.xpu.current_device()} + else: + raise RuntimeError("No GPU found. A GPU is needed for quantization.") + logger.info("The device_map was not initialized.Setting device_map to `{'':torch.cuda.current_device()}`.") + + if isinstance(device_map, str): + if device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]: + raise ValueError( + "If passing a string for `device_map`, please choose 'auto', 'balanced', 'balanced_low_0' or " + "'sequential'." + ) + + special_dtypes = {} + special_dtypes.update( + { + name: bnb_quantization_config.torch_dtype + for name, _ in model.named_parameters() + if any(m in name for m in bnb_quantization_config.skip_modules) + } + ) + special_dtypes.update( + { + name: torch.float32 + for name, _ in model.named_parameters() + if any(m in name for m in bnb_quantization_config.keep_in_fp32_modules) + } + ) + + kwargs = {} + kwargs["special_dtypes"] = special_dtypes + kwargs["no_split_module_classes"] = no_split_module_classes + kwargs["dtype"] = bnb_quantization_config.target_dtype + + # get max_memory for each device. + if device_map != "sequential": + max_memory = get_balanced_memory( + model, + low_zero=(device_map == "balanced_low_0"), + max_memory=max_memory, + **kwargs, + ) + + kwargs["max_memory"] = max_memory + device_map = infer_auto_device_map(model, **kwargs) + + if isinstance(device_map, dict): + # check if don't have any quantized module on the cpu + modules_not_to_convert = bnb_quantization_config.skip_modules + bnb_quantization_config.keep_in_fp32_modules + + device_map_without_some_modules = { + key: device_map[key] for key in device_map.keys() if key not in modules_not_to_convert + } + for device in ["cpu", "disk"]: + if device in device_map_without_some_modules.values(): + if bnb_quantization_config.load_in_4bit: + raise ValueError( + """ + Some modules are dispatched on the CPU or the disk. Make sure you have enough GPU RAM to fit + the quantized model. If you want to dispatch the model on the CPU or the disk while keeping + these modules in `torch_dtype`, you need to pass a custom `device_map` to + `load_and_quantize_model`. Check + https://huggingface.co/docs/accelerate/main/en/usage_guides/quantization#offload-modules-to-cpu-and-disk + for more details. + """ + ) + else: + logger.info( + "Some modules are are offloaded to the CPU or the disk. Note that these modules will be converted to 8-bit" + ) + del device_map_without_some_modules + return device_map + + +def replace_with_bnb_layers(model, bnb_quantization_config, modules_to_not_convert=None, current_key_name=None): + """ + A helper function to replace all `torch.nn.Linear` modules by `bnb.nn.Linear8bit` modules or by `bnb.nn.Linear4bit` + modules from the `bitsandbytes`library. The function will be run recursively and replace `torch.nn.Linear` modules. + + Parameters: + model (`torch.nn.Module`): + Input model or `torch.nn.Module` as the function is run recursively. + modules_to_not_convert (`List[str]`): + Names of the modules to not quantize convert. In practice we keep the `lm_head` in full precision for + numerical stability reasons. + current_key_name (`List[str]`, *optional*): + An array to track the current key of the recursion. This is used to check whether the current key (part of + it) is not in the list of modules to not convert. + """ + + if modules_to_not_convert is None: + modules_to_not_convert = [] + + model, has_been_replaced = _replace_with_bnb_layers( + model, bnb_quantization_config, modules_to_not_convert, current_key_name + ) + if not has_been_replaced: + logger.warning( + "You are loading your model in 8bit or 4bit but no linear modules were found in your model." + " this can happen for some architectures such as gpt2 that uses Conv1D instead of Linear layers." + " Please double check your model architecture, or submit an issue on github if you think this is" + " a bug." + ) + return model + + +def _replace_with_bnb_layers( + model, + bnb_quantization_config, + modules_to_not_convert=None, + current_key_name=None, +): + """ + Private method that wraps the recursion for module replacement. + + Returns the converted model and a boolean that indicates if the conversion has been successful or not. + """ + # bitsandbytes will initialize device(e.g. CUDA, XPU) on import, so it needs to be imported lazily + import bitsandbytes as bnb + + has_been_replaced = False + for name, module in model.named_children(): + if current_key_name is None: + current_key_name = [] + current_key_name.append(name) + if isinstance(module, nn.Linear) and name not in modules_to_not_convert: + # Check if the current key is not in the `modules_to_not_convert` + current_key_name_str = ".".join(current_key_name) + proceed = True + for key in modules_to_not_convert: + if ( + (key in current_key_name_str) and (key + "." in current_key_name_str) + ) or key == current_key_name_str: + proceed = False + break + if proceed: + # Load bnb module with empty weight and replace ``nn.Linear` module + if bnb_quantization_config.load_in_8bit: + bnb_module = bnb.nn.Linear8bitLt( + module.in_features, + module.out_features, + module.bias is not None, + has_fp16_weights=False, + threshold=bnb_quantization_config.llm_int8_threshold, + ) + elif bnb_quantization_config.load_in_4bit: + bnb_module = bnb.nn.Linear4bit( + module.in_features, + module.out_features, + module.bias is not None, + bnb_quantization_config.bnb_4bit_compute_dtype, + compress_statistics=bnb_quantization_config.bnb_4bit_use_double_quant, + quant_type=bnb_quantization_config.bnb_4bit_quant_type, + ) + else: + raise ValueError("load_in_8bit and load_in_4bit can't be both False") + bnb_module.weight.data = module.weight.data + if module.bias is not None: + bnb_module.bias.data = module.bias.data + bnb_module.requires_grad_(False) + setattr(model, name, bnb_module) + has_been_replaced = True + if len(list(module.children())) > 0: + _, _has_been_replaced = _replace_with_bnb_layers( + module, bnb_quantization_config, modules_to_not_convert, current_key_name + ) + has_been_replaced = has_been_replaced | _has_been_replaced + # Remove the last key for recursion + current_key_name.pop(-1) + return model, has_been_replaced + + +def get_keys_to_not_convert(model): + r""" + An utility function to get the key of the module to keep in full precision if any For example for CausalLM modules + we may want to keep the lm_head in full precision for numerical stability reasons. For other architectures, we want + to keep the tied weights of the model. The function will return a list of the keys of the modules to not convert in + int8. + + Parameters: + model (`torch.nn.Module`): + Input model + """ + # Create a copy of the model + with init_empty_weights(): + tied_model = deepcopy(model) # this has 0 cost since it is done inside `init_empty_weights` context manager` + + tied_params = find_tied_parameters(tied_model) + # For compatibility with Accelerate < 0.18 + if isinstance(tied_params, dict): + tied_keys = sum(list(tied_params.values()), []) + list(tied_params.keys()) + else: + tied_keys = sum(tied_params, []) + has_tied_params = len(tied_keys) > 0 + + # Check if it is a base model + is_base_model = False + if hasattr(model, "base_model_prefix"): + is_base_model = not hasattr(model, model.base_model_prefix) + + # Ignore this for base models (BertModel, GPT2Model, etc.) + if (not has_tied_params) and is_base_model: + return [] + + # otherwise they have an attached head + list_modules = list(model.named_children()) + list_last_module = [list_modules[-1][0]] + + # add last module together with tied weights + intersection = set(list_last_module) - set(tied_keys) + list_untouched = list(set(tied_keys)) + list(intersection) + + # remove ".weight" from the keys + names_to_remove = [".weight", ".bias"] + filtered_module_names = [] + for name in list_untouched: + for name_to_remove in names_to_remove: + if name_to_remove in name: + name = name.replace(name_to_remove, "") + filtered_module_names.append(name) + + return filtered_module_names + + +def has_4bit_bnb_layers(model): + """Check if we have `bnb.nn.Linear4bit` or `bnb.nn.Linear8bitLt` layers inside our model""" + # bitsandbytes will initialize device(e.g. CUDA, XPU) on import, so it needs to be imported lazily + import bitsandbytes as bnb + + for m in model.modules(): + if isinstance(m, bnb.nn.Linear4bit): + return True + return False + + +def get_parameter_device(parameter: nn.Module): + return next(parameter.parameters()).device + + +def quantize_and_offload_8bit(model, param, param_name, new_dtype, offload_folder, offload_index, fp16_statistics): + # if it is not quantized, we quantize and offload the quantized weights and the SCB stats + if fp16_statistics is None: + set_module_tensor_to_device(model, param_name, 0, dtype=new_dtype, value=param) + tensor_name = param_name + module = model + if "." in tensor_name: + splits = tensor_name.split(".") + for split in splits[:-1]: + new_module = getattr(module, split) + if new_module is None: + raise ValueError(f"{module} has no attribute {split}.") + module = new_module + tensor_name = splits[-1] + # offload weights + module._parameters[tensor_name].requires_grad = False + offload_weight(module._parameters[tensor_name], param_name, offload_folder, index=offload_index) + if hasattr(module._parameters[tensor_name], "SCB"): + offload_weight( + module._parameters[tensor_name].SCB, + param_name.replace("weight", "SCB"), + offload_folder, + index=offload_index, + ) + else: + offload_weight(param, param_name, offload_folder, index=offload_index) + offload_weight(fp16_statistics, param_name.replace("weight", "SCB"), offload_folder, index=offload_index) + + set_module_tensor_to_device(model, param_name, "meta", dtype=new_dtype, value=torch.empty(*param.size())) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/constants.py b/kbidu_env/Lib/site-packages/accelerate/utils/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..0eba7bc3d61cb012366f43a31873871f6d29a83d --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/constants.py @@ -0,0 +1,108 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import operator as op + +import torch + + +SCALER_NAME = "scaler.pt" +MODEL_NAME = "pytorch_model" +SAFE_MODEL_NAME = "model" +RNG_STATE_NAME = "random_states" +OPTIMIZER_NAME = "optimizer" +SCHEDULER_NAME = "scheduler" +SAMPLER_NAME = "sampler" +PROFILE_PATTERN_NAME = "profile_{suffix}.json" +WEIGHTS_NAME = f"{MODEL_NAME}.bin" +WEIGHTS_PATTERN_NAME = "pytorch_model{suffix}.bin" +WEIGHTS_INDEX_NAME = f"{WEIGHTS_NAME}.index.json" +SAFE_WEIGHTS_NAME = f"{SAFE_MODEL_NAME}.safetensors" +SAFE_WEIGHTS_PATTERN_NAME = "model{suffix}.safetensors" +SAFE_WEIGHTS_INDEX_NAME = f"{SAFE_WEIGHTS_NAME}.index.json" +SAGEMAKER_PYTORCH_VERSION = "1.10.2" +SAGEMAKER_PYTHON_VERSION = "py38" +SAGEMAKER_TRANSFORMERS_VERSION = "4.17.0" +SAGEMAKER_PARALLEL_EC2_INSTANCES = ["ml.p3.16xlarge", "ml.p3dn.24xlarge", "ml.p4dn.24xlarge"] +FSDP_SHARDING_STRATEGY = ["FULL_SHARD", "SHARD_GRAD_OP", "NO_SHARD", "HYBRID_SHARD", "HYBRID_SHARD_ZERO2"] +FSDP_AUTO_WRAP_POLICY = ["TRANSFORMER_BASED_WRAP", "SIZE_BASED_WRAP", "NO_WRAP"] +FSDP_BACKWARD_PREFETCH = ["BACKWARD_PRE", "BACKWARD_POST", "NO_PREFETCH"] +FSDP_STATE_DICT_TYPE = ["FULL_STATE_DICT", "LOCAL_STATE_DICT", "SHARDED_STATE_DICT"] +FSDP2_STATE_DICT_TYPE = ["SHARDED_STATE_DICT", "FULL_STATE_DICT"] +FSDP_PYTORCH_VERSION = ( + "2.1.0.a0+32f93b1" # Technically should be 2.1.0, but MS-AMP uses this specific prerelease in their Docker image. +) +FSDP2_PYTORCH_VERSION = "2.6.0" +DTENSOR_PYTORCH_VERSION = "2.5.0" +FSDP_MODEL_NAME = "pytorch_model_fsdp" +DEEPSPEED_MULTINODE_LAUNCHERS = ["pdsh", "standard", "openmpi", "mvapich", "mpich", "nossh", "slurm"] +TORCH_DYNAMO_MODES = ["default", "reduce-overhead", "max-autotune"] +ELASTIC_LOG_LINE_PREFIX_TEMPLATE_PYTORCH_VERSION = "2.2.0" +XPU_PROFILING_AVAILABLE_PYTORCH_VERSION = "2.4.0" +MITA_PROFILING_AVAILABLE_PYTORCH_VERSION = "2.1.0" +BETA_TP_AVAILABLE_PYTORCH_VERSION = "2.3.0" + +BETA_TP_AVAILABLE_TRANSFORMERS_VERSION = "4.52.0" +BETA_CP_AVAILABLE_PYTORCH_VERSION = "2.6.0" +BETA_SP_AVAILABLE_DEEPSPEED_VERSION = "0.18.2" + +STR_OPERATION_TO_FUNC = {">": op.gt, ">=": op.ge, "==": op.eq, "!=": op.ne, "<=": op.le, "<": op.lt} + +# These are the args for `torch.distributed.launch` for pytorch < 1.9 +TORCH_LAUNCH_PARAMS = [ + "nnodes", + "nproc_per_node", + "rdzv_backend", + "rdzv_endpoint", + "rdzv_id", + "rdzv_conf", + "standalone", + "max_restarts", + "monitor_interval", + "start_method", + "role", + "module", + "m", + "no_python", + "run_path", + "log_dir", + "r", + "redirects", + "t", + "tee", + "node_rank", + "master_addr", + "master_port", +] + +CUDA_DISTRIBUTED_TYPES = ["DEEPSPEED", "MULTI_GPU", "FSDP", "MEGATRON_LM", "TP"] +TORCH_DISTRIBUTED_OPERATION_TYPES = CUDA_DISTRIBUTED_TYPES + [ + "MULTI_NPU", + "MULTI_MLU", + "MULTI_SDAA", + "MULTI_MUSA", + "MULTI_XPU", + "MULTI_CPU", + "MULTI_HPU", + "MULTI_NEURON", +] +SUPPORTED_PYTORCH_LAYERS_FOR_UPCASTING = ( + torch.nn.Conv1d, + torch.nn.Conv2d, + torch.nn.Conv3d, + torch.nn.ConvTranspose1d, + torch.nn.ConvTranspose2d, + torch.nn.ConvTranspose3d, + torch.nn.Linear, +) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/dataclasses.py b/kbidu_env/Lib/site-packages/accelerate/utils/dataclasses.py new file mode 100644 index 0000000000000000000000000000000000000000..ab57092c1f75f81690ab8e3b2b816c642fb064b7 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/dataclasses.py @@ -0,0 +1,3226 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +General namespace and dataclass related classes +""" + +import argparse +import copy +import enum +import functools +import logging +import os +import warnings +from collections.abc import Iterable +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import timedelta +from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Union, get_args + +import torch + +from .constants import ( + BETA_CP_AVAILABLE_PYTORCH_VERSION, + BETA_TP_AVAILABLE_PYTORCH_VERSION, + BETA_TP_AVAILABLE_TRANSFORMERS_VERSION, + FSDP2_PYTORCH_VERSION, + FSDP_AUTO_WRAP_POLICY, + FSDP_BACKWARD_PREFETCH, + FSDP_SHARDING_STRATEGY, + MITA_PROFILING_AVAILABLE_PYTORCH_VERSION, + XPU_PROFILING_AVAILABLE_PYTORCH_VERSION, +) +from .environment import parse_flag_from_env, str_to_bool +from .imports import ( + is_cuda_available, + is_hpu_available, + is_mlu_available, + is_msamp_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_rocm_available, + is_torchao_available, + is_transformer_engine_available, + is_xpu_available, +) +from .versions import compare_versions, is_torch_version + + +if TYPE_CHECKING: + # Mock imports for type checking + from torchao.float8 import Float8LinearConfig + + +logger = logging.getLogger(__name__) + + +class KwargsHandler: + """ + Internal mixin that implements a `to_kwargs()` method for a dataclass. + """ + + def to_dict(self): + return copy.deepcopy(self.__dict__) + + def to_kwargs(self): + """ + Returns a dictionary containing the attributes with values different from the default of this class. + """ + # import clear_environment here to avoid circular import problem + from .environment import clear_environment + + with clear_environment(): + default_dict = self.__class__().to_dict() + this_dict = self.to_dict() + return {k: v for k, v in this_dict.items() if default_dict[k] != v} + + +class EnumWithContains(enum.EnumMeta): + "A metaclass that adds the ability to check if `self` contains an item with the `in` operator" + + def __contains__(cls, item): + try: + cls(item) + except ValueError: + return False + return True + + +class BaseEnum(enum.Enum, metaclass=EnumWithContains): + "An enum class that can get the value of an item with `str(Enum.key)`" + + def __str__(self): + return self.value + + @classmethod + def list(cls): + "Method to list all the possible items in `cls`" + return list(map(str, cls)) + + +@dataclass +class AutocastKwargs(KwargsHandler): + """ + Use this object in your [`Accelerator`] to customize how `torch.autocast` behaves. Please refer to the + documentation of this [context manager](https://pytorch.org/docs/stable/amp.html#torch.autocast) for more + information on each argument. + + Example: + + ```python + from accelerate import Accelerator + from accelerate.utils import AutocastKwargs + + kwargs = AutocastKwargs(cache_enabled=True) + accelerator = Accelerator(kwargs_handlers=[kwargs]) + ``` + """ + + enabled: bool = True + cache_enabled: Optional[bool] = None + + +class DDPCommunicationHookType(BaseEnum): + """ + Represents a type of communication hook used in DDP. + + Values: + + - **NO** -- no communication hook + - **FP16** -- DDP communication hook to compress the gradients in FP16 + - **BF16** -- DDP communication hook to compress the gradients in BF16 + - **POWER_SGD** -- DDP communication hook to use PowerSGD + - **BATCHED_POWER_SGD** -- DDP communication hook to use batched PowerSGD + """ + + NO = "no" + FP16 = "fp16" + BF16 = "bf16" + POWER_SGD = "power_sgd" + BATCHED_POWER_SGD = "batched_power_sgd" + + +@dataclass +class DistributedDataParallelKwargs(KwargsHandler): + """ + Use this object in your [`Accelerator`] to customize how your model is wrapped in a + `torch.nn.parallel.DistributedDataParallel`. Please refer to the documentation of this + [wrapper](https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html) for more + information on each argument. + + + + `gradient_as_bucket_view` is only available in PyTorch 1.7.0 and later versions. + + `static_graph` is only available in PyTorch 1.11.0 and later versions. + + + + Example: + + ```python + from accelerate import Accelerator + from accelerate.utils import DistributedDataParallelKwargs + + kwargs = DistributedDataParallelKwargs(find_unused_parameters=True) + accelerator = Accelerator(kwargs_handlers=[kwargs]) + ``` + """ + + dim: int = 0 + broadcast_buffers: bool = True + bucket_cap_mb: int = 25 + find_unused_parameters: bool = False + check_reduction: bool = False + gradient_as_bucket_view: bool = False + static_graph: bool = False + + comm_hook: DDPCommunicationHookType = DDPCommunicationHookType.NO + comm_wrapper: Literal[ + DDPCommunicationHookType.NO, + DDPCommunicationHookType.FP16, + DDPCommunicationHookType.BF16, + ] = DDPCommunicationHookType.NO + comm_state_option: dict = field(default_factory=dict) + + def to_dict(self, ignore_keys=("comm_hook", "comm_wrapper", "comm_state_option")): + return {k: v for k, v in super().to_dict().items() if k not in ignore_keys} + + def register_comm_hook(self, model): + from torch.distributed.algorithms.ddp_comm_hooks import ( + default_hooks, + powerSGD_hook, + ) + + hook_map: dict[DDPCommunicationHookType, Callable] = { + DDPCommunicationHookType.FP16: default_hooks.fp16_compress_hook, + DDPCommunicationHookType.BF16: default_hooks.bf16_compress_hook, + DDPCommunicationHookType.POWER_SGD: powerSGD_hook.powerSGD_hook, + DDPCommunicationHookType.BATCHED_POWER_SGD: powerSGD_hook.batched_powerSGD_hook, + } + + wrapper_map: dict[DDPCommunicationHookType, Callable] = { + DDPCommunicationHookType.FP16: default_hooks.fp16_compress_wrapper, + DDPCommunicationHookType.BF16: default_hooks.bf16_compress_wrapper, + } + + hook: Optional[Callable] = hook_map.get(self.comm_hook) + wrapper: Optional[Callable] = wrapper_map.get(self.comm_wrapper) + + if hook and wrapper: + hook = wrapper(hook) + + if hook: + state = ( + powerSGD_hook.PowerSGDState(None, **self.comm_state_option) + if self.comm_hook + in ( + DDPCommunicationHookType.POWER_SGD, + DDPCommunicationHookType.BATCHED_POWER_SGD, + ) + else None + ) + model.register_comm_hook( + state=state, + hook=hook, + ) + + +@dataclass +class GradScalerKwargs(KwargsHandler): + """ + Use this object in your [`Accelerator`] to customize the behavior of mixed precision, specifically how the + `torch.amp.GradScaler` or `torch.cuda.amp.GradScaler` used is created. Please refer to the documentation of this + [scaler](https://pytorch.org/docs/stable/amp.html?highlight=gradscaler) for more information on each argument. + + + + `torch.cuda.amp.GradScaler` is only available in PyTorch 1.5.0 and later versions, and `torch.amp.GradScaler` is + only available in PyTorch 2.4.0 and later versions. + + + + Example: + + ```python + from accelerate import Accelerator + from accelerate.utils import GradScalerKwargs + + kwargs = GradScalerKwargs(backoff_factor=0.25) + accelerator = Accelerator(kwargs_handlers=[kwargs]) + ``` + """ + + init_scale: float = 65536.0 + growth_factor: float = 2.0 + backoff_factor: float = 0.5 + growth_interval: int = 2000 + enabled: bool = True + + +@dataclass +class InitProcessGroupKwargs(KwargsHandler): + """ + Use this object in your [`Accelerator`] to customize the initialization of the distributed processes. Please refer + to the documentation of this + [method](https://pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group) for more + information on each argument. + + Note: If `timeout` is set to `None`, the default will be based upon how `backend` is set. + + ```python + from datetime import timedelta + from accelerate import Accelerator + from accelerate.utils import InitProcessGroupKwargs + + kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=800)) + accelerator = Accelerator(kwargs_handlers=[kwargs]) + ``` + """ + + backend: Optional[str] = "nccl" + init_method: Optional[str] = None + timeout: Optional[timedelta] = None + + def __post_init__(self): + if self.timeout is None: + seconds = 1800 if self.backend != "nccl" else 600 + self.timeout = timedelta(seconds=seconds) + + +# Literals +Backend = Literal["MSAMP", "TE"] +OptLevel = Literal["O1", "O2"] +FP8Format = Literal["HYBRID", "E4M3", "E5M2"] +AmaxComputeAlgorithm = Literal["max", "most_recent"] + + +# FP8 training recipe kwargs +@dataclass +class AORecipeKwargs(KwargsHandler): + """ + Use this object in your [`Accelerator`] to customize the initialization of the recipe for FP8 mixed precision + training with `torchao` FP8. + + Args: + config (`torchao.float8.Float8LinearConfig`, *optional*, default to `None`): + The configuration for the FP8 training. If `None`, a default config will be created with sensible + defaults for most use cases: + - `pad_inner_dim=True`: Pads matrix dimensions to be divisible by 16, required for `torch._scaled_mm` + operations to prevent runtime errors. + - `enable_fsdp_float8_all_gather=True`: Enables FP8 all-gather for FSDP2. This provides memory bandwidth + savings by casting parameters before the all-gather operation, saving 50% bandwidth compared to BF16. + + You can override these defaults by providing your own `Float8LinearConfig` instance. + module_filter_func (`Callable`, *optional*, default to `None`): + Optional function that must take in a module and layer name, and returns a boolean indicating whether the + module should be converted to FP8. Defaults to `accelerate.utils.ao.filter_linear_layers`. See it for an + example. + """ + + config: Optional["Float8LinearConfig"] = None + module_filter_func: Optional[Callable] = None + pad_inner_dim: Optional[bool] = None + enable_fsdp_float8_all_gather: Optional[bool] = None + + def __post_init__(self): + env_prefix = "ACCELERATE_FP8_" + if not is_torchao_available(): + raise ImportError("TorchAO is not available. Please install it or use a different backend.") + + if self.config is None: + from torchao.float8 import Float8LinearConfig + + # Check environment variables for overrides + if self.pad_inner_dim is None: + self.pad_inner_dim = parse_flag_from_env(env_prefix + "PAD_INNER_DIM", default=True) + if self.enable_fsdp_float8_all_gather is None: + self.enable_fsdp_float8_all_gather = parse_flag_from_env( + env_prefix + "ENABLE_FSDP_FLOAT8_ALL_GATHER", default=True + ) + self.config = Float8LinearConfig( + pad_inner_dim=self.pad_inner_dim, + enable_fsdp_float8_all_gather=self.enable_fsdp_float8_all_gather, + ) + + +@dataclass +class TERecipeKwargs(KwargsHandler): + """ + Use this object in your [`Accelerator`] to customize the initialization of the recipe for FP8 mixed precision + training with `transformer-engine`. + + + + For more information on the args, please refer to the API + [documentation](https://docs.nvidia.com/deeplearning/transformer-engine/user-guide/api/common.html). + + + + ```python + from accelerate import Accelerator + from accelerate.utils import TERecipeKwargs + + kwargs = TERecipeKwargs(fp8_format="HYBRID") + accelerator = Accelerator(mixed_precision="fp8", kwargs_handlers=[kwargs]) + ``` + + Args: + use_autocast_during_eval (`bool`, *optional*, default to `False`): + Whether to use FP8 autocast during eval mode. Generally better metrics are found when this is `False`. + margin (`int`, *optional*, default to 0): + The margin to use for the gradient scaling. + interval (`int`, *optional*, default to 1): + The interval to use for how often the scaling factor is recomputed. + fp8_format (`str`, *optional*, default to "HYBRID"): + The format to use for the FP8 recipe. Must be one of `HYBRID`, `E4M3` or `E5M2`. (Generally `HYBRID` for + training, `E4M3` or `E5M2` for evaluation) + amax_history_len (`int`, *optional*, default to 1024): + The length of the history to use for the scaling factor computation + amax_compute_algo (`str`, *optional*, default to "most_recent"): + The algorithm to use for the scaling factor computation. Must be one of `max` or `most_recent`. + override_linear_precision (`tuple` of three `bool`, *optional*, default to `(False, False, False)`): + Whether or not to execute `fprop`, `dgrad`, and `wgrad` GEMMS in higher precision. + """ + + use_autocast_during_eval: Optional[bool] = None + margin: Optional[int] = None + interval: Optional[int] = None + fp8_format: FP8Format = None + amax_history_len: Optional[int] = None + amax_compute_algo: AmaxComputeAlgorithm = None + override_linear_precision: tuple[bool, bool, bool] = None + use_mxfp8_block_scaling: Optional[bool] = None + + def __post_init__(self): + env_prefix = "ACCELERATE_FP8_" + if not is_transformer_engine_available(): + raise ImportError("TransformerEngine is not available. Please install it or use a different backend.") + if self.use_autocast_during_eval is None: + self.use_autocast_during_eval = parse_flag_from_env(env_prefix + "USE_AUTOCAST_DURING_EVAL") + if self.margin is None: + self.margin = int(os.environ.get(env_prefix + "MARGIN", 0)) + if self.interval is None: + self.interval = int(os.environ.get(env_prefix + "INTERVAL", 1)) + if self.fp8_format is None: + self.fp8_format = os.environ.get(env_prefix + "FORMAT", "HYBRID") + self.fp8_format = self.fp8_format.upper() + if self.fp8_format not in get_args(FP8Format): + raise ValueError(f"`fp8_format` must be one of {' or '.join(get_args(FP8Format))}.") + if self.amax_compute_algo is None: + self.amax_compute_algo = os.environ.get(env_prefix + "AMAX_COMPUTE_ALGO", "most_recent") + self.amax_compute_algo = self.amax_compute_algo.lower() + if self.amax_compute_algo not in get_args(AmaxComputeAlgorithm): + raise ValueError(f"`amax_compute_algo` must be one of {' or '.join(get_args(AmaxComputeAlgorithm))}") + if self.amax_history_len is None: + self.amax_history_len = int(os.environ.get(env_prefix + "AMAX_HISTORY_LEN", 1024)) + if self.override_linear_precision is None: + fprop = parse_flag_from_env(env_prefix + "OVERRIDE_FPROP") + dgrad = parse_flag_from_env(env_prefix + "OVERRIDE_DGRAD") + wgrad = parse_flag_from_env(env_prefix + "OVERRIDE_WGRAD") + self.override_linear_precision = (fprop, dgrad, wgrad) + if self.use_mxfp8_block_scaling is None: + self.use_mxfp8_block_scaling = parse_flag_from_env(env_prefix + "USE_MXFP8_BLOCK_SCALING") + + +@dataclass +class MSAMPRecipeKwargs(KwargsHandler): + """ + Use this object in your [`Accelerator`] to customize the initialization of the recipe for FP8 mixed precision + training with `ms-amp`. + """ + + opt_level: OptLevel = None + + def __post_init__(self): + env_prefix = "ACCELERATE_FP8_" + if self.opt_level is None: + self.opt_level = os.environ.get(env_prefix + "OPT_LEVEL", "O2") + if self.opt_level not in get_args(OptLevel): + raise ValueError(f"`opt_level` must be one of {' or '.join(get_args(OptLevel))}") + + +@dataclass +class FP8RecipeKwargs(TERecipeKwargs, MSAMPRecipeKwargs): + """ + Deprecated. Please use one of the proper FP8 recipe kwargs classes such as `TERecipeKwargs` or `MSAMPRecipeKwargs` + instead. + """ + + backend: Backend = None + + def __post_init__(self): + env_prefix = "ACCELERATE_FP8_" + warnings.warn( + "FP8RecipeKwargs is deprecated and will be removed in Accelerate v2.0.0. " + "Please use one of the proper FP8 recipe kwargs classes such as TERecipeKwargs or MSAMPRecipeKwargs instead.", + FutureWarning, + ) + default_backend = "msamp" if is_msamp_available() else "te" + if self.backend is None: + self.backend = os.environ.get(env_prefix + "BACKEND", default_backend) + self.backend = self.backend.upper() + if self.backend not in get_args(Backend): + raise ValueError("`backend` must be 'MSAMP' or 'TE' (TransformerEngine) to use `FP8RecipeKwargs`.") + super().__post_init__() + + +# Literal +ProfilerActivity = Literal["cpu", "xpu", "mtia", "cuda", "hpu"] + + +@dataclass +class ProfileKwargs(KwargsHandler): + """ + Use this object in your [`Accelerator`] to customize the initialization of the profiler. Please refer to the + documentation of this [context manager](https://pytorch.org/docs/stable/profiler.html#torch.profiler.profile) for + more information on each argument. + + + + `torch.profiler` is only available in PyTorch 1.8.1 and later versions. + + + + Example: + + ```python + from accelerate import Accelerator + from accelerate.utils import ProfileKwargs + + kwargs = ProfileKwargs(activities=["cpu", "cuda"]) + accelerator = Accelerator(kwargs_handlers=[kwargs]) + ``` + + Args: + activities (`List[str]`, *optional*, default to `None`): + The list of activity groups to use in profiling. Must be one of `"cpu"`, `"xpu"`, `"mtia"`, "hpu" or + `"cuda"`. + schedule_option (`Dict[str, int]`, *optional*, default to `None`): + The schedule option to use for the profiler. Available keys are `wait`, `warmup`, `active`, `repeat` and + `skip_first`. The profiler will skip the first `skip_first` steps, then wait for `wait` steps, then do the + warmup for the next `warmup` steps, then do the active recording for the next `active` steps and then + repeat the cycle starting with `wait` steps. The optional number of cycles is specified with the `repeat` + parameter, the zero value means that the cycles will continue until the profiling is finished. + on_trace_ready (`Callable`, *optional*, default to `None`): + Callable that is called at each step when schedule returns `ProfilerAction.RECORD_AND_SAVE` during the + profiling. + record_shapes (`bool`, *optional*, default to `False`): + Save information about operator’s input shapes. + profile_memory (`bool`, *optional*, default to `False`): + Track tensor memory allocation/deallocation + with_stack (`bool`, *optional*, default to `False`): + Record source information (file and line number) for the ops. + with_flops (`bool`, *optional*, default to `False`): + Use formula to estimate the FLOPS of specific operators + with_modules (`bool`, *optional*, default to `False`): + Record module hierarchy (including function names) corresponding to the callstack of the op. + output_trace_dir (`str`, *optional*, default to `None`): + Exports the collected trace in Chrome JSON format. Chrome use 'chrome://tracing' view json file. Defaults + to None, which means profiling does not store json files. + """ + + activities: Optional[list[ProfilerActivity]] = None + schedule_option: Optional[dict[str, int]] = None + on_trace_ready: Optional[Callable] = None + record_shapes: bool = False + profile_memory: bool = False + with_stack: bool = False + with_flops: bool = False + with_modules: bool = False + output_trace_dir: Optional[str] = None + + def _get_profiler_activity(self, activity: ProfilerActivity) -> torch.profiler.ProfilerActivity: + """Get the profiler activity from the string. + + Args: + activity (str): The profiler activity name. + + Returns: + torch.profiler.ProfilerActivity: The profiler activity. + """ + + profiler_activity_map: dict[str, torch.profiler.ProfilerActivity] = { + "cpu": torch.profiler.ProfilerActivity.CPU, + "cuda": torch.profiler.ProfilerActivity.CUDA, + } + + if is_hpu_available(): + profiler_activity_map["hpu"] = torch.profiler.ProfilerActivity.HPU + + if is_torch_version(">=", XPU_PROFILING_AVAILABLE_PYTORCH_VERSION): + if torch.xpu.is_available(): + profiler_activity_map["xpu"] = torch.profiler.ProfilerActivity.XPU + + if is_torch_version(">=", MITA_PROFILING_AVAILABLE_PYTORCH_VERSION): + if torch.mtia.is_available(): + profiler_activity_map["mtia"] = torch.profiler.ProfilerActivity.MTIA + + if activity not in profiler_activity_map: + raise ValueError(f"Invalid profiler activity: {activity}. Must be one of {list(profiler_activity_map)}.") + return profiler_activity_map[activity] + + def build(self) -> torch.profiler.profile: + """ + Build a profiler object with the current configuration. + + Returns: + torch.profiler.profile: The profiler object. + """ + activities: Optional[list[ProfilerActivity]] = None + if self.activities is not None: + activities = [self._get_profiler_activity(activity) for activity in self.activities] + schedule: Optional[torch.profiler.schedule] = None + if self.schedule_option is not None: + schedule = torch.profiler.schedule(**self.schedule_option) + + return torch.profiler.profile( + activities=activities, + schedule=schedule, + on_trace_ready=self.on_trace_ready, + record_shapes=self.record_shapes, + profile_memory=self.profile_memory, + with_stack=self.with_stack, + with_flops=self.with_flops, + with_modules=self.with_modules, + ) + + +class DistributedType(str, enum.Enum): + """ + Represents a type of distributed environment. + + Values: + + - **NO** -- Not a distributed environment, just a single process. + - **MULTI_CPU** -- Distributed on multiple CPU nodes. + - **MULTI_GPU** -- Distributed on multiple GPUs. + - **MULTI_MLU** -- Distributed on multiple MLUs. + - **MULTI_SDAA** -- Distributed on multiple SDAAs. + - **MULTI_MUSA** -- Distributed on multiple MUSAs. + - **MULTI_NPU** -- Distributed on multiple NPUs. + - **MULTI_XPU** -- Distributed on multiple XPUs. + - **MULTI_HPU** -- Distributed on multiple HPUs. + - **MULTI_NEURON** -- Distributed on multiple Neuron cores. + - **DEEPSPEED** -- Using DeepSpeed. + - **FSDP** -- Using Fully Sharded Data Parallelism (FSDP). + - **XLA** -- Using TorchXLA. + - **MEGATRON_LM** -- Using Megatron-LM. + """ + + # Subclassing str as well as Enum allows the `DistributedType` to be JSON-serializable out of the box. + NO = "NO" + MULTI_CPU = "MULTI_CPU" + MULTI_GPU = "MULTI_GPU" + MULTI_NPU = "MULTI_NPU" + MULTI_MLU = "MULTI_MLU" + MULTI_SDAA = "MULTI_SDAA" + MULTI_MUSA = "MULTI_MUSA" + MULTI_XPU = "MULTI_XPU" + DEEPSPEED = "DEEPSPEED" + FSDP = "FSDP" + XLA = "XLA" + MEGATRON_LM = "MEGATRON_LM" + MULTI_HPU = "MULTI_HPU" + MULTI_NEURON = "MULTI_NEURON" + + +class SageMakerDistributedType(str, enum.Enum): + """ + Represents a type of distributed environment. + + Values: + + - **NO** -- Not a distributed environment, just a single process. + - **DATA_PARALLEL** -- using sagemaker distributed data parallelism. + - **MODEL_PARALLEL** -- using sagemaker distributed model parallelism. + """ + + # Subclassing str as well as Enum allows the `SageMakerDistributedType` to be JSON-serializable out of the box. + NO = "NO" + DATA_PARALLEL = "DATA_PARALLEL" + MODEL_PARALLEL = "MODEL_PARALLEL" + + +class FP8BackendType(str, enum.Enum): + """ + Represents the backend used for FP8. + + Values: + + - **TE** -- using TransformerEngine. + - **MSAMP** -- using msamp. + """ + + # Subclassing str as well as Enum allows the `FP8BackendType` to be JSON-serializable out of the box. + NO = "NO" + TE = "TE" + MSAMP = "MSAMP" + AO = "AO" + + +class ComputeEnvironment(str, enum.Enum): + """ + Represents a type of the compute environment. + + Values: + + - **LOCAL_MACHINE** -- private/custom cluster hardware. + - **AMAZON_SAGEMAKER** -- Amazon SageMaker as compute environment. + """ + + # Subclassing str as well as Enum allows the `ComputeEnvironment` to be JSON-serializable out of the box. + LOCAL_MACHINE = "LOCAL_MACHINE" + AMAZON_SAGEMAKER = "AMAZON_SAGEMAKER" + + +class DynamoBackend(str, BaseEnum): + """ + Represents a dynamo backend (see https://pytorch.org/docs/stable/torch.compiler.html). + + Values: + + - **NO** -- Do not use torch dynamo. + - **EAGER** -- Uses PyTorch to run the extracted GraphModule. This is quite useful in debugging TorchDynamo + issues. + - **AOT_EAGER** -- Uses AotAutograd with no compiler, i.e, just using PyTorch eager for the AotAutograd's + extracted forward and backward graphs. This is useful for debugging, and unlikely to give speedups. + - **INDUCTOR** -- Uses TorchInductor backend with AotAutograd and cudagraphs by leveraging codegened Triton + kernels. [Read + more](https://dev-discuss.pytorch.org/t/torchinductor-a-pytorch-native-compiler-with-define-by-run-ir-and-symbolic-shapes/747) + - **AOT_TS_NVFUSER** -- nvFuser with AotAutograd/TorchScript. [Read + more](https://dev-discuss.pytorch.org/t/tracing-with-primitives-update-1-nvfuser-and-its-primitives/593) + - **NVPRIMS_NVFUSER** -- nvFuser with PrimTorch. [Read + more](https://dev-discuss.pytorch.org/t/tracing-with-primitives-update-1-nvfuser-and-its-primitives/593) + - **CUDAGRAPHS** -- cudagraphs with AotAutograd. [Read more](https://github.com/pytorch/torchdynamo/pull/757) + - **OFI** -- Uses Torchscript optimize_for_inference. Inference only. [Read + more](https://pytorch.org/docs/stable/generated/torch.jit.optimize_for_inference.html) + - **FX2TRT** -- Uses Nvidia TensorRT for inference optimizations. Inference only. [Read + more](https://github.com/pytorch/TensorRT/blob/master/docsrc/tutorials/getting_started_with_fx_path.rst) + - **ONNXRT** -- Uses ONNXRT for inference on CPU/GPU. Inference only. [Read more](https://onnxruntime.ai/) + - **TENSORRT** -- Uses ONNXRT to run TensorRT for inference optimizations. [Read + more](https://github.com/onnx/onnx-tensorrt) + - **AOT_TORCHXLA_TRACE_ONCE** -- Uses Pytorch/XLA with TorchDynamo optimization, for training. [Read + more](https://github.com/pytorch/xla/blob/r2.0/docs/dynamo.md) + - **TORCHXLA_TRACE_ONCE** -- Uses Pytorch/XLA with TorchDynamo optimization, for inference. [Read + more](https://github.com/pytorch/xla/blob/r2.0/docs/dynamo.md) + - **TVM** -- Uses Apache TVM for inference optimizations. [Read more](https://tvm.apache.org/) + - **HPU_BACKEND** -- Uses HPU backend for inference optimizations. + + """ + + # Subclassing str as well as Enum allows the `SageMakerDistributedType` to be JSON-serializable out of the box. + NO = "NO" + EAGER = "EAGER" + AOT_EAGER = "AOT_EAGER" + INDUCTOR = "INDUCTOR" + AOT_TS_NVFUSER = "AOT_TS_NVFUSER" + NVPRIMS_NVFUSER = "NVPRIMS_NVFUSER" + CUDAGRAPHS = "CUDAGRAPHS" + OFI = "OFI" + FX2TRT = "FX2TRT" + ONNXRT = "ONNXRT" + TENSORRT = "TENSORRT" + AOT_TORCHXLA_TRACE_ONCE = "AOT_TORCHXLA_TRACE_ONCE" + TORCHXLA_TRACE_ONCE = "TORCHXLA_TRACE_ONCE" + TVM = "TVM" + HPU_BACKEND = "HPU_BACKEND" + + +class LoggerType(BaseEnum): + """Represents a type of supported experiment tracker + + Values: + + - **ALL** -- all available trackers in the environment that are supported + - **TENSORBOARD** -- TensorBoard as an experiment tracker + - **WANDB** -- wandb as an experiment tracker + - **TRACKIO** -- trackio as an experiment tracker + - **COMETML** -- comet_ml as an experiment tracker + - **MLFLOW** -- mlflow as an experiment tracker + - **CLEARML** -- clearml as an experiment tracker + - **DVCLIVE** -- dvclive as an experiment tracker + - **SWANLAB** -- swanlab as an experiment tracker + """ + + ALL = "all" + AIM = "aim" + TENSORBOARD = "tensorboard" + WANDB = "wandb" + TRACKIO = "trackio" + COMETML = "comet_ml" + MLFLOW = "mlflow" + CLEARML = "clearml" + DVCLIVE = "dvclive" + SWANLAB = "swanlab" + + +class PrecisionType(str, BaseEnum): + """Represents a type of precision used on floating point values + + Values: + + - **NO** -- using full precision (FP32) + - **FP16** -- using half precision + - **BF16** -- using brain floating point precision + """ + + NO = "no" + FP8 = "fp8" + FP16 = "fp16" + BF16 = "bf16" + + +class RNGType(BaseEnum): + TORCH = "torch" + CUDA = "cuda" + MLU = "mlu" + SDAA = "sdaa" + MUSA = "musa" + NPU = "npu" + XLA = "xla" + XPU = "xpu" + HPU = "hpu" + NEURON = "neuron" + GENERATOR = "generator" + + +class CustomDtype(enum.Enum): + r""" + An enum that contains multiple custom dtypes that can be used for `infer_auto_device_map`. + """ + + FP8 = "fp8" + INT4 = "int4" + INT2 = "int2" + + +# data classes + + +@dataclass +class TensorInformation: + shape: torch.Size + dtype: torch.dtype + + +@dataclass +class DataLoaderConfiguration: + """ + Configuration for dataloader-related items when calling `accelerator.prepare`. + + Args: + split_batches (`bool`, defaults to `False`): + Whether or not the accelerator should split the batches yielded by the dataloaders across the devices. If + `True`, the actual batch size used will be the same on any kind of distributed processes, but it must be a + round multiple of `num_processes` you are using. If `False`, actual batch size used will be the one set in + your script multiplied by the number of processes. + dispatch_batches (`bool`, defaults to `None`): + If set to `True`, the dataloader prepared by the Accelerator is only iterated through on the main process + and then the batches are split and broadcast to each process. Will default to `True` for `DataLoader` whose + underlying dataset is an `IterableDataset`, `False` otherwise. + even_batches (`bool`, defaults to `True`): + If set to `True`, in cases where the total batch size across all processes does not exactly divide the + dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among + all workers. + use_seedable_sampler (`bool`, defaults to `False`): + Whether or not use a fully seedable random sampler ([`data_loader.SeedableRandomSampler`]). Ensures + training results are fully reproducible using a different sampling technique. While seed-to-seed results + may differ, on average the differences are negligible when using multiple different seeds to compare. + Should also be ran with [`~utils.set_seed`] for the best results. + data_seed (`int`, defaults to `None`): + The seed to use for the underlying generator when using `use_seedable_sampler`. If `None`, the generator + will use the current default seed from torch. + non_blocking (`bool`, defaults to `False`): + If set to `True`, the dataloader prepared by the Accelerator will utilize non-blocking host-to-device + transfers, allowing for better overlap between dataloader communication and computation. Recommended that + the prepared dataloader has `pin_memory` set to `True` to work properly. + use_stateful_dataloader (`bool`, defaults to `False`): + If set to `True`, the dataloader prepared by the Accelerator will be backed by + [torchdata.StatefulDataLoader](https://github.com/pytorch/data/tree/main/torchdata/stateful_dataloader). + This requires `torchdata` version 0.8.0 or higher that supports StatefulDataLoader to be installed. + """ + + split_batches: bool = field( + default=False, + metadata={ + "help": "Whether or not the accelerator should split the batches yielded by the dataloaders across the devices. If" + " `True` the actual batch size used will be the same on any kind of distributed processes, but it must be a" + " round multiple of the `num_processes` you are using. If `False`, actual batch size used will be the one set" + " in your script multiplied by the number of processes." + }, + ) + dispatch_batches: bool = field( + default=None, + metadata={ + "help": "If set to `True`, the dataloader prepared by the Accelerator is only iterated through on the main process" + " and then the batches are split and broadcast to each process. Will default to `True` for `DataLoader` whose" + " underlying dataset is an `IterableDataset`, `False` otherwise." + }, + ) + even_batches: bool = field( + default=True, + metadata={ + "help": "If set to `True`, in cases where the total batch size across all processes does not exactly divide the" + " dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among" + " all workers." + }, + ) + use_seedable_sampler: bool = field( + default=False, + metadata={ + "help": "Whether or not use a fully seedable random sampler ([`data_loader.SeedableRandomSampler`])." + "Ensures training results are fully reproducible using a different sampling technique. " + "While seed-to-seed results may differ, on average the differences are negligible when using" + "multiple different seeds to compare. Should also be ran with [`~utils.set_seed`] for the best results." + }, + ) + data_seed: int = field( + default=None, + metadata={ + "help": "The seed to use for the underlying generator when using `use_seedable_sampler`. If `None`, the generator" + " will use the current default seed from torch." + }, + ) + non_blocking: bool = field( + default=False, + metadata={ + "help": "If set to `True`, the dataloader prepared by the Accelerator will utilize non-blocking host-to-device" + " transfers, allowing for better overlap between dataloader communication and computation. Recommended that the" + " prepared dataloader has `pin_memory` set to `True` to work properly." + }, + ) + use_stateful_dataloader: bool = field( + default=False, + metadata={ + "help": "If set to `True`, the dataloader prepared by the Accelerator will be backed by " + "[torchdata.StatefulDataLoader](https://github.com/pytorch/data/tree/main/torchdata/stateful_dataloader). This requires `torchdata` version 0.8.0 or higher that supports StatefulDataLoader to be installed." + }, + ) + + +@dataclass +class ProjectConfiguration: + """ + Configuration for the Accelerator object based on inner-project needs. + + Args: + project_dir (`str`, defaults to `None`): + A path to a directory for storing data. + logging_dir (`str`, defaults to `None`): + A path to a directory for storing logs of locally-compatible loggers. If None, defaults to `project_dir`. + automatic_checkpoint_naming (`bool`, defaults to `False`): + Whether saved states should be automatically iteratively named. + total_limit (`int`, defaults to `None`): + The maximum number of total saved states to keep. + iteration (`int`, defaults to `0`): + The current save iteration. + save_on_each_node (`bool`, defaults to `False`): + When doing multi-node distributed training, whether to save models and checkpoints on each node, or only on + the main one. + """ + + project_dir: str = field(default=None, metadata={"help": "A path to a directory for storing data."}) + logging_dir: str = field( + default=None, + metadata={ + "help": "A path to a directory for storing logs of locally-compatible loggers. If None, defaults to `project_dir`." + }, + ) + automatic_checkpoint_naming: bool = field( + default=False, + metadata={"help": "Whether saved states should be automatically iteratively named."}, + ) + + total_limit: int = field( + default=None, + metadata={"help": "The maximum number of total saved states to keep."}, + ) + + iteration: int = field( + default=0, + metadata={"help": "The current save iteration."}, + ) + + save_on_each_node: bool = field( + default=False, + metadata={ + "help": ( + "When doing multi-node distributed training, whether to save models and checkpoints on each node, or" + " only on the main one" + ) + }, + ) + + def set_directories(self, project_dir: Optional[str] = None): + "Sets `self.project_dir` and `self.logging_dir` to the appropriate values." + self.project_dir = project_dir + if self.logging_dir is None: + self.logging_dir = project_dir + + def __post_init__(self): + self.set_directories(self.project_dir) + + +@dataclass +class GradientAccumulationPlugin(KwargsHandler): + """ + A plugin to configure gradient accumulation behavior. You can only pass one of `gradient_accumulation_plugin` or + `gradient_accumulation_steps` to [`Accelerator`]. Passing both raises an error. + + Parameters: + num_steps (`int`): + The number of steps to accumulate gradients for. + adjust_scheduler (`bool`, *optional*, defaults to `True`): + Whether to adjust the scheduler steps to account for the number of steps being accumulated. Should be + `True` if the used scheduler was not adjusted for gradient accumulation. + sync_with_dataloader (`bool`, *optional*, defaults to `True`): + Whether to synchronize setting the gradients when at the end of the dataloader. + sync_each_batch (`bool`, *optional*): + Whether to synchronize setting the gradients at each data batch. Setting to `True` may reduce memory + requirements when using gradient accumulation with distributed training, at expense of speed. + + Example: + + ```python + from accelerate.utils import GradientAccumulationPlugin + + gradient_accumulation_plugin = GradientAccumulationPlugin(num_steps=2) + accelerator = Accelerator(gradient_accumulation_plugin=gradient_accumulation_plugin) + ``` + """ + + num_steps: int = field( + default=None, + metadata={"help": "The number of steps to accumulate gradients for."}, + ) + adjust_scheduler: bool = field( + default=True, + metadata={ + "help": "Whether to adjust the scheduler steps to account for the number of steps being accumulated. Should be `True` if the used scheduler was not adjusted for gradient accumulation." + }, + ) + sync_with_dataloader: bool = field( + default=True, + metadata={ + "help": "Whether to synchronize setting the gradients when at the end of the dataloader. Should only be set to `False` if you know what you're doing." + }, + ) + sync_each_batch: bool = field( + default=False, + metadata={ + "help": "Whether to synchronize setting the gradients at each data batch. Setting to `True` may reduce memory requirements when using gradient accumulation with distributed training, at expense of speed." + }, + ) + + +@dataclass +class TorchDynamoPlugin(KwargsHandler): + """ + This plugin is used to compile a model with PyTorch 2.0 + + Args: + backend (`DynamoBackend`, defaults to `None`): + A valid Dynamo backend. See https://pytorch.org/docs/stable/torch.compiler.html for more details. + mode (`str`, defaults to `None`): + Possible options are 'default', 'reduce-overhead' or 'max-autotune'. + fullgraph (`bool`, defaults to `None`): + Whether it is ok to break model into several subgraphs. + dynamic (`bool`, defaults to `None`): + Whether to use dynamic shape for tracing. + options (`Any`, defaults to `None`): + A dictionary of options to pass to the backend. + disable (`bool`, defaults to `False`): + Turn torch.compile() into a no-op for testing + use_regional_compilation (`bool`, defaults to `None`): + Use it to reduce the cold start compilation time of torch.compile() by targeting repeated blocks of the + same class and compiling them sequentially to hit the compiler's cache. For example, in `GPT2LMHeadModel`, + the repeated block/class is `GPT2Block`, and can be accessed as `model.transformer.h[0]`. The rest of the + model (e.g model.lm_head) is compiled separately. + """ + + backend: DynamoBackend = field( + default=None, + metadata={"help": f"Possible options are {[b.value.lower() for b in DynamoBackend]}"}, + ) + mode: str = field( + default=None, + metadata={"help": "Possible options are 'default', 'reduce-overhead' or 'max-autotune'"}, + ) + fullgraph: bool = field( + default=None, + metadata={"help": "Whether it is ok to break model into several subgraphs"}, + ) + dynamic: bool = field(default=None, metadata={"help": "Whether to use dynamic shape for tracing"}) + options: Any = field( + default=None, + metadata={"help": "A dictionary of options to pass to the backend."}, + ) + disable: bool = field( + default=False, + metadata={"help": "Turn torch.compile() into a no-op for testing"}, + ) + + use_regional_compilation: bool = field( + default=None, + metadata={ + "help": ( + # https://pytorch.org/tutorials/recipes/regional_compilation.html + "Use it to reduce the cold start compilation time of torch.compile() by targeting repeated " + "blocks of the same class and compiling them sequentially to hit the compiler's cache. For " + "example, in `GPT2LMHeadModel`, the repeated block/class is `GPT2Block`, and can be accessed " + "as `model.transformer.h[0]`. The rest of the model (e.g model.lm_head) is compiled separately." + ) + }, + ) + + def __post_init__(self): + prefix = "ACCELERATE_DYNAMO_" + if self.backend is None: + self.backend = os.environ.get(prefix + "BACKEND", "no") + self.backend = DynamoBackend(self.backend.upper()) + + if self.mode is None: + self.mode = os.environ.get(prefix + "MODE", "default") + if self.fullgraph is None: + self.fullgraph = str_to_bool(os.environ.get(prefix + "USE_FULLGRAPH", "False")) == 1 + if self.use_regional_compilation is None: + self.use_regional_compilation = ( + str_to_bool(os.environ.get(prefix + "USE_REGIONAL_COMPILATION", "False")) == 1 + ) + + if self.dynamic is None and os.environ.get(prefix + "USE_DYNAMIC", None) is not None: + self.dynamic = str_to_bool(os.environ.get(prefix + "USE_DYNAMIC", "False")) == 1 + + def to_dict(self): + dynamo_config = copy.deepcopy(self.__dict__) + dynamo_config["backend"] = dynamo_config["backend"].value.lower() + return dynamo_config + + def to_kwargs(self): + kwargs = super().to_kwargs() + kwargs.pop("use_regional_compilation", None) + return kwargs + + +@dataclass +class DeepSpeedPlugin: + """ + This plugin is used to integrate DeepSpeed. + + Args: + hf_ds_config (`Any`, defaults to `None`): + Path to DeepSpeed config file or dict or an object of class `accelerate.utils.deepspeed.HfDeepSpeedConfig`. + gradient_accumulation_steps (`int`, defaults to `None`): + Number of steps to accumulate gradients before updating optimizer states. If not set, will use the value + from the `Accelerator` directly. + gradient_clipping (`float`, defaults to `None`): + Enable gradient clipping with value. + zero_stage (`int`, defaults to `None`): + Possible options are 0, 1, 2, 3. Default will be taken from environment variable. + is_train_batch_min (`bool`, defaults to `True`): + If both train & eval dataloaders are specified, this will decide the `train_batch_size`. + offload_optimizer_device (`str`, defaults to `None`): + Possible options are none|cpu|nvme. Only applicable with ZeRO Stages 2 and 3. + offload_param_device (`str`, defaults to `None`): + Possible options are none|cpu|nvme. Only applicable with ZeRO Stage 3. + offload_optimizer_nvme_path (`str`, defaults to `None`): + Possible options are /nvme|/local_nvme. Only applicable with ZeRO Stage 3. + offload_param_nvme_path (`str`, defaults to `None`): + Possible options are /nvme|/local_nvme. Only applicable with ZeRO Stage 3. + zero3_init_flag (`bool`, defaults to `None`): + Flag to indicate whether to save 16-bit model. Only applicable with ZeRO Stage-3. + zero3_save_16bit_model (`bool`, defaults to `None`): + Flag to indicate whether to save 16-bit model. Only applicable with ZeRO Stage-3. + transformer_moe_cls_names (`str`, defaults to `None`): + Comma-separated list of Transformers MoE layer class names (case-sensitive). For example, + `MixtralSparseMoeBlock`, `Qwen2MoeSparseMoeBlock`, `JetMoEAttention`, `JetMoEBlock`, etc. + enable_msamp (`bool`, defaults to `None`): + Flag to indicate whether to enable MS-AMP backend for FP8 training. + msasmp_opt_level (`Optional[Literal["O1", "O2"]]`, defaults to `None`): + Optimization level for MS-AMP (defaults to 'O1'). Only applicable if `enable_msamp` is True. Should be one + of ['O1' or 'O2']. + """ + + hf_ds_config: Any = field( + default=None, + metadata={ + "help": "path to DeepSpeed config file or dict or an object of class `accelerate.utils.deepspeed.HfDeepSpeedConfig`." + }, + ) + gradient_accumulation_steps: int = field( + default=None, + metadata={ + "help": "Number of steps to accumulate gradients before updating optimizer states. If not set, will use the value from the `Accelerator` directly." + }, + ) + gradient_clipping: float = field(default=None, metadata={"help": "Enable gradient clipping with value"}) + zero_stage: int = field( + default=None, + metadata={"help": "Possible options are 0,1,2,3; Default will be taken from environment variable"}, + ) + is_train_batch_min: bool = field( + default=True, + metadata={"help": "If both train & eval dataloaders are specified, this will decide the train_batch_size"}, + ) + offload_optimizer_device: str = field( + default=None, + metadata={"help": "Possible options are none|cpu|nvme. Only applicable with ZeRO Stages 2 and 3."}, + ) + offload_param_device: str = field( + default=None, + metadata={"help": "Possible options are none|cpu|nvme. Only applicable with ZeRO Stage 3."}, + ) + offload_optimizer_nvme_path: str = field( + default=None, + metadata={"help": "Possible options are /nvme|/local_nvme. Only applicable with ZeRO Stage 3."}, + ) + offload_param_nvme_path: str = field( + default=None, + metadata={"help": "Possible options are /nvme|/local_nvme. Only applicable with ZeRO Stage 3."}, + ) + zero3_init_flag: bool = field( + default=None, + metadata={ + "help": "Flag to indicate whether to enable `deepspeed.zero.Init` for constructing massive models." + "Only applicable with ZeRO Stage-3." + }, + ) + zero3_save_16bit_model: bool = field( + default=None, + metadata={"help": "Flag to indicate whether to save 16-bit model. Only applicable with ZeRO Stage-3."}, + ) + transformer_moe_cls_names: str = field( + default=None, + metadata={ + "help": "comma-separated list of transformers MoE layer class names (case-sensitive), e.g : " + " `MixtralSparseMoeBlock`, `Qwen2MoeSparseMoeBlock`, `JetMoEAttention,JetMoEBlock` ..." + }, + ) + enable_msamp: bool = field( + default=None, + metadata={"help": "Flag to indicate whether to enable MS-AMP backend for FP8 training."}, + ) + msamp_opt_level: Optional[Literal["O1", "O2"]] = field( + default=None, + metadata={ + "help": "Optimization level for MS-AMP (defaults to 'O1'). Only applicable if `enable_msamp` is True. Should be one of ['O1' or 'O2']." + }, + ) + + def __post_init__(self): + from .deepspeed import HfDeepSpeedConfig + + if self.gradient_accumulation_steps is None: + gas = os.environ.get("ACCELERATE_GRADIENT_ACCUMULATION_STEPS", "auto") + self.gradient_accumulation_steps = int(gas) if gas.isdigit() else gas + + if self.gradient_clipping is None: + gradient_clipping = os.environ.get("ACCELERATE_GRADIENT_CLIPPING", "auto") + self.gradient_clipping = gradient_clipping if gradient_clipping == "auto" else float(gradient_clipping) + + if self.zero_stage is None: + self.zero_stage = int(os.environ.get("ACCELERATE_DEEPSPEED_ZERO_STAGE", 2)) + + if self.offload_optimizer_device is None: + self.offload_optimizer_device = os.environ.get("ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE", "none") + + if self.offload_param_device is None: + self.offload_param_device = os.environ.get("ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICE", "none") + + if self.offload_optimizer_nvme_path is None: + self.offload_optimizer_nvme_path = os.environ.get( + "ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_NVME_PATH", "none" + ) + + if self.offload_param_nvme_path is None: + self.offload_param_nvme_path = os.environ.get("ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_NVME_PATH", "none") + + if self.zero3_save_16bit_model is None: + self.zero3_save_16bit_model = ( + os.environ.get("ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL", "false").lower() == "true" + ) + if self.enable_msamp is None: + self.enable_msamp = os.environ.get("ACCELERATE_FP8_BACKEND", None) == "MSAMP" + + if self.msamp_opt_level is None: + self.msamp_opt_level = os.environ.get("ACCELERATE_FP8_OPT_LEVEL", "O1") + + if self.hf_ds_config is None: + self.hf_ds_config = os.environ.get("ACCELERATE_DEEPSPEED_CONFIG_FILE", "none") + + if ( + isinstance(self.hf_ds_config, dict) + or (isinstance(self.hf_ds_config, str) and self.hf_ds_config != "none") + or isinstance(self.hf_ds_config, HfDeepSpeedConfig) + ): + if not isinstance(self.hf_ds_config, HfDeepSpeedConfig): + self.hf_ds_config = HfDeepSpeedConfig(self.hf_ds_config) + if "gradient_accumulation_steps" not in self.hf_ds_config.config: + self.hf_ds_config.config["gradient_accumulation_steps"] = 1 + if "zero_optimization" not in self.hf_ds_config.config: + raise ValueError("Please specify the ZeRO optimization config in the DeepSpeed config.") + + self._deepspeed_config_checks() + plugin_to_config_mapping = { + "gradient_accumulation_steps": "gradient_accumulation_steps", + "gradient_clipping": "gradient_clipping", + "zero_stage": "zero_optimization.stage", + "offload_optimizer_device": "zero_optimization.offload_optimizer.device", + "offload_param_device": "zero_optimization.offload_param.device", + "offload_param_nvme_path": "zero_optimization.offload_param.nvme_path", + "offload_optimizer_nvme_path": "zero_optimization.offload_optimizer.nvme_path", + "zero3_save_16bit_model": "zero_optimization.stage3_gather_16bit_weights_on_model_save", + } + kwargs = {v: getattr(self, k) for k, v in plugin_to_config_mapping.items() if getattr(self, k) is not None} + for key in kwargs.keys(): + self.fill_match(key, **kwargs, must_match=False) + self.hf_ds_config.set_stage_and_offload() + + # filling the missing values in the class attributes from the DeepSpeed config + # when using the DeepSpeed config file. + for key, value in plugin_to_config_mapping.items(): + config_value = self.hf_ds_config.get_value(value) + if config_value is not None and config_value != "auto": + setattr(self, key, config_value) + else: + config = { + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "gradient_accumulation_steps": self.gradient_accumulation_steps, + "zero_optimization": { + "stage": self.zero_stage, + "offload_optimizer": { + "device": self.offload_optimizer_device, + "nvme_path": ( + self.offload_optimizer_nvme_path if self.offload_optimizer_device == "nvme" else None + ), + }, + "offload_param": { + "device": self.offload_param_device, + "nvme_path": (self.offload_param_nvme_path if self.offload_param_device == "nvme" else None), + }, + "stage3_gather_16bit_weights_on_model_save": self.zero3_save_16bit_model, + }, + } + if self.gradient_clipping: + config["gradient_clipping"] = self.gradient_clipping + self.hf_ds_config = HfDeepSpeedConfig(config) + + self.deepspeed_config = self.hf_ds_config.config + self.deepspeed_config["steps_per_print"] = float("inf") # this will stop deepspeed from logging @ stdout + if self.zero3_init_flag is None: + self.zero3_init_flag = ( + str_to_bool( + os.environ.get( + "ACCELERATE_DEEPSPEED_ZERO3_INIT", + str(self.hf_ds_config.is_zero3()), + ) + ) + == 1 + ) + if self.zero3_init_flag and not self.hf_ds_config.is_zero3(): + warnings.warn("DeepSpeed Zero3 Init flag is only applicable for ZeRO Stage 3. Setting it to False.") + self.zero3_init_flag = False + # NOTE: Set to False by default, will be set to `True` automatically if it's the first plugin passed + # to the `Accelerator`'s `deepspeed_plugin` param, *or* `AcceleratorState().enable_deepspeed_plugin(plugin_key)` is manually called + self._set_selected(False) + + # Ignore if it's already set + if self.enable_msamp and "msamp" not in self.deepspeed_config: + if self.zero_stage == 3: + raise NotImplementedError( + "MS-AMP is not supported for ZeRO Stage 3. Please use ZeRO Stage 0, 1, or 2 instead." + ) + if self.msamp_opt_level not in ["O1", "O2"]: + raise ValueError("Invalid optimization level for MS-AMP. Please use one of ['O1' or'O2'].") + self.deepspeed_config["msamp"] = { + "enabled": True, + "opt_level": self.msamp_opt_level, + } + + def fill_match(self, ds_key_long, mismatches=None, must_match=True, **kwargs): + mismatches = [] if mismatches is None else mismatches + config, ds_key = self.hf_ds_config.find_config_node(ds_key_long) + if config is None: + return + + if config.get(ds_key) == "auto": + if ds_key_long in kwargs: + config[ds_key] = kwargs[ds_key_long] + return + else: + raise ValueError( + f"`{ds_key_long}` not found in kwargs. " + f"Please specify `{ds_key_long}` without `auto` (set to correct value) in the DeepSpeed config file or " + "pass it in kwargs." + ) + + if not must_match: + return + + ds_val = config.get(ds_key) + if ds_val is not None and ds_key_long in kwargs: + if ds_val != kwargs[ds_key_long]: + mismatches.append(f"- ds {ds_key_long}={ds_val} vs arg {ds_key_long}={kwargs[ds_key_long]}") + + def is_auto(self, ds_key_long): + val = self.hf_ds_config.get_value(ds_key_long) + if val is None: + return False + else: + return val == "auto" + + def get_value(self, ds_key_long, default=None): + return self.hf_ds_config.get_value(ds_key_long, default) + + def deepspeed_config_process(self, prefix="", mismatches=None, config=None, must_match=True, **kwargs): + """Process the DeepSpeed config with the values from the kwargs.""" + mismatches = [] if mismatches is None else mismatches + if config is None: + config = self.deepspeed_config + for key, value in config.items(): + if isinstance(value, dict): + self.deepspeed_config_process( + prefix=prefix + key + ".", + mismatches=mismatches, + config=value, + must_match=must_match, + **kwargs, + ) + else: + self.fill_match(prefix + key, mismatches, must_match=must_match, **kwargs) + if len(mismatches) > 0 and prefix == "": + mismatches_msg = "\n".join(mismatches) + raise ValueError( + "Please correct the following DeepSpeed config values that mismatch kwargs " + f" values:\n{mismatches_msg}\nThe easiest method is to set these DeepSpeed config values to 'auto'." + ) + + def set_mixed_precision(self, mixed_precision): + ds_config = self.deepspeed_config + kwargs = { + "fp16.enabled": mixed_precision == "fp16", + # When training in fp8, we still rely on bf16 autocast for the core mixed precision + "bf16.enabled": mixed_precision in ("bf16", "fp8"), + } + if mixed_precision == "fp16": + if "fp16" not in ds_config: + ds_config["fp16"] = {"enabled": True, "auto_cast": True} + elif mixed_precision in ("bf16", "fp8"): + if "bf16" not in ds_config: + ds_config["bf16"] = {"enabled": True} + + if mixed_precision == "fp8" and self.enable_msamp: + if "msamp" not in ds_config: + ds_config["msamp"] = { + "enabled": True, + "opt_level": self.msamp_opt_level, + } + + if mixed_precision != "no": + diff_dtype = "bf16" if mixed_precision == "fp16" else "fp16" + if str(ds_config.get(diff_dtype, {}).get("enabled", "False")).lower() == "true": + raise ValueError( + f"`--mixed_precision` arg cannot be set to `{mixed_precision}` when `{diff_dtype}` is set in the DeepSpeed config file." + ) + for dtype in ["fp16", "bf16"]: + if dtype not in ds_config: + ds_config[dtype] = {"enabled": False} + self.fill_match("fp16.enabled", must_match=False, **kwargs) + self.fill_match("bf16.enabled", must_match=False, **kwargs) + + # On ROCm, bf16 DeepSpeed training can silently produce NaN weights because + # bf16 has no NaN/Inf safety net (unlike fp16 loss scaling). Accumulating + # gradients in fp32 for the collective avoids the overflow path. + if mixed_precision in ("bf16", "fp8") and is_rocm_available() and "communication_data_type" not in ds_config: + ds_config["communication_data_type"] = "fp32" + logger.info( + "ROCm + DeepSpeed + bf16 detected: setting " + "`communication_data_type='fp32'` to avoid bf16 overflow corrupting " + "weights. Set it explicitly in your DeepSpeed config to override." + ) + + def set_deepspeed_weakref(self): + from .imports import is_transformers_available + + ds_config = copy.deepcopy(self.deepspeed_config) + if self.zero3_init_flag: + if not is_transformers_available(): + raise Exception( + "When `zero3_init_flag` is set, it requires Transformers to be installed. " + "Please run `pip install transformers`." + ) + if "gradient_accumulation_steps" not in ds_config or ds_config["gradient_accumulation_steps"] == "auto": + ds_config["gradient_accumulation_steps"] = 1 + if "train_micro_batch_size_per_gpu" not in ds_config or ds_config["train_micro_batch_size_per_gpu"] == "auto": + ds_config["train_micro_batch_size_per_gpu"] = 1 + if ds_config.get("train_batch_size", None) == "auto": + del ds_config["train_batch_size"] + + if compare_versions("transformers", "<", "4.46"): + from transformers.deepspeed import ( + HfDeepSpeedConfig, + unset_hf_deepspeed_config, + ) + else: + from transformers.integrations import ( + HfDeepSpeedConfig, + unset_hf_deepspeed_config, + ) + + unset_hf_deepspeed_config() + self.dschf = HfDeepSpeedConfig(ds_config) # keep this object alive # noqa + + def is_zero3_init_enabled(self): + return self.zero3_init_flag + + @contextmanager + def zero3_init_context_manager(self, enable=False): + old = self.zero3_init_flag + if old == enable: + yield + else: + self.zero3_init_flag = enable + self.dschf = None + self.set_deepspeed_weakref() + yield + self.zero3_init_flag = old + self.dschf = None + self.set_deepspeed_weakref() + + def _deepspeed_config_checks(self): + env_variable_names_to_ignore = [ + "ACCELERATE_GRADIENT_ACCUMULATION_STEPS", + "ACCELERATE_GRADIENT_CLIPPING", + "ACCELERATE_DEEPSPEED_ZERO_STAGE", + "ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE", + "ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICE", + "ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_NVME_PATH", + "ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_NVME_PATH", + "ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL", + "ACCELERATE_MIXED_PRECISION", + ] + env_variable_names_to_ignore = [ + name.replace("ACCELERATE_", "").replace("DEEPSPEED_", "").lower() for name in env_variable_names_to_ignore + ] + + deepspeed_fields_from_accelerate_config = os.environ.get("ACCELERATE_CONFIG_DS_FIELDS", "").split(",") + + if any(name in env_variable_names_to_ignore for name in deepspeed_fields_from_accelerate_config): + raise ValueError( + f"When using `deepspeed_config_file`, the following accelerate config variables will be ignored: {env_variable_names_to_ignore}.\n" + "Please specify them appropriately in the DeepSpeed config file.\n" + "If you are using an accelerate config file, remove others config variables mentioned in the above specified list.\n" + "The easiest method is to create a new config following the questionnaire via `accelerate config`.\n" + "It will only ask for the necessary config variables when using `deepspeed_config_file`." + ) + + def set_moe_leaf_modules(self, model): + if self.transformer_moe_cls_names is None: + self.transformer_moe_cls_names = os.environ.get("ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES", None) + if self.transformer_moe_cls_names is not None: + if compare_versions("deepspeed", "<", "0.14.0"): + raise ImportError("DeepSpeed version must be >= 0.14.0 to use MOE support. Please update DeepSpeed.") + from deepspeed.utils import set_z3_leaf_modules + + class_names = self.transformer_moe_cls_names.split(",") + transformer_moe_cls = [] + for layer_class in class_names: + transformer_cls = get_module_class_from_name(model, layer_class) + if transformer_cls is None: + raise Exception( + f"Could not find a transformer layer class called '{layer_class}' to wrap in the model." + ) + else: + transformer_moe_cls.append(transformer_cls) + set_z3_leaf_modules(model, transformer_moe_cls) # z3_leaf + + def select(self, _from_accelerator_state: bool = False): + """ + Sets the HfDeepSpeedWeakref to use the current deepspeed plugin configuration + """ + if not _from_accelerator_state: + raise ValueError( + "A `DeepSpeedPlugin` object must be enabled manually by calling `AcceleratorState().enable_deepspeed_plugin(plugin_key)`." + ) + self.set_deepspeed_weakref() + self._set_selected(True) + + def _unselect(self): + self._set_selected(False) + + def _set_selected(self, value: bool): + """ + Private setter for the 'enabled' attribute. + """ + self._selected = value + + @property + def selected(self): + return self._selected + + @selected.setter + def selected(self, value): + raise NotImplementedError( + "'enabled' can only be set through calling 'AcceleratorState().enable_deepspeed_plugin(key)'." + ) + + +@dataclass +class FullyShardedDataParallelPlugin: + """ + This plugin is used to enable fully sharded data parallelism. + + Args: + fsdp_version (`int`, defaults to `1`): + The version of FSDP to use. Defaults to 1. If set to 2, launcher expects the config to be converted to + FSDP2 format. + sharding_strategy (`Union[str, torch.distributed.fsdp.ShardingStrategy]`, defaults to `'FULL_SHARD'`): + Sharding strategy to use. Should be either a `str` or an instance of + `torch.distributed.fsdp.fully_sharded_data_parallel.ShardingStrategy`. Is deprecated in favor of + `reshard_after_forward`. + reshard_after_forward (`Union[str, torch.distributed.fsdp.ShardingStrategy, bool]`, defaults to `'FULL_SHARD'` for `fsdp_version=1` and `True` for `fsdp_version=2`): + Sharding strategy to use. Should be a bool if `fsdp_version` is set to 2 else a `str` or an instance of + `torch.distributed.fsdp.fully_sharded_data_parallel.ShardingStrategy`. + backward_prefetch (`Union[str, torch.distributed.fsdp.BackwardPrefetch]`, defaults to `'NO_PREFETCH'`): + Backward prefetch strategy to use. Should be either a `str` or an instance of + `torch.distributed.fsdp.fully_sharded_data_parallel.BackwardPrefetch`. + mixed_precision_policy (`Optional[Union[dict, str, torch.distributed.fsdp.MixedPrecision, torch.distributed.fsdp.MixedPrecisionPolicy]]`, defaults to `None`): + A config to enable mixed precision training with FullyShardedDataParallel. If passing in a `dict`, it + should have the following keys: `param_dtype`, `reduce_dtype`, and `buffer_dtype`, can be an instance of + `torch.distributed.fsdp.MixedPrecisionPolicy` if `fsdp_version` is set to 2. If passing in a `str`, it + should be one of the following values: fp8, fp16, bf16, fp32, and used to set `param_dtype`, + `reduce_dtype`, and `buffer_dtype`. + auto_wrap_policy (`Optional(Union[Callable, Literal["transformer_based_wrap", "size_based_wrap", "no_wrap"]]), defaults to `NO_WRAP`): + A callable or string specifying a policy to recursively wrap layers with FSDP. If a string, it must be one + of `transformer_based_wrap`, `size_based_wrap`, or `no_wrap`. See + `torch.distributed.fsdp.wrap.size_based_wrap_policy` for a direction on what it should look like. + cpu_offload (`Union[bool, torch.distributed.fsdp.CPUOffload, torch.distributed.fsdp.CPUOffloadPolicy]`, defaults to `False`): + Whether to offload parameters to CPU. Should be either a `bool` or an instance of + `torch.distributed.fsdp.fully_sharded_data_parallel.CPUOffload` or + `torch.distributed.fsdp.fully_sharded_data_parallel.CPUOffloadPolicy` if `fsdp_version` is set to 2. + ignored_modules (`Optional[Union[Iterable[torch.nn.Module], str]]`, defaults to `None`): + A list of modules to ignore when wrapping with FSDP. When passing a string, will match the modules by name + using regex fullmatch. If `fsdp_version` is set to 2, the modules are converted to parameters and used. + state_dict_type (`Union[str, torch.distributed.fsdp.StateDictType]`, defaults to `'FULL_STATE_DICT'`): + State dict type to use. If a string, it must be one of `full_state_dict`, `local_state_dict`, or + `sharded_state_dict`. + state_dict_config (`Optional[Union[torch.distributed.fsdp.FullStateDictConfig, torch.distributed.fsdp.ShardedStateDictConfig]`, defaults to `None`): + State dict config to use. Is determined based on the `state_dict_type` if not passed in. + optim_state_dict_config (`Optional[Union[torch.distributed.fsdp.FullOptimStateDictConfig, torch.distributed.fsdp.ShardedOptimStateDictConfig]`, defaults to `None`): + Optim state dict config to use. Is determined based on the `state_dict_type` if not passed in. + limit_all_gathers (`bool`, defaults to `True`): + Whether to have FSDP explicitly synchronizes the CPU thread to prevent too many in-flight all-gathers. This + bool only affects the sharded strategies that schedule all-gathers. Enabling this can help lower the number + of CUDA malloc retries. + use_orig_params (`bool`, defaults to `False`): + Whether to use the original parameters for the optimizer. + param_init_fn (`Optional[Callable[[torch.nn.Module], None]`, defaults to `None`): + A `Callable[torch.nn.Module] -> None` that specifies how modules that are currently on the meta device + should be initialized onto an actual device. Only applicable when `sync_module_states` is `True`. By + default is a `lambda` which calls `to_empty` on the module. + sync_module_states (`bool`, defaults to `False`): + Whether each individually wrapped FSDP unit should broadcast module parameters from rank 0 to ensure they + are the same across all ranks after initialization. Defaults to `False` unless `cpu_ram_efficient_loading` + is `True`, then will be forcibly enabled. + forward_prefetch (`bool`, defaults to `False`): + Whether to have FSDP explicitly prefetches the next upcoming all-gather while executing in the forward + pass. only use with Static graphs. + activation_checkpointing (`bool`, defaults to `False`): + A technique to reduce memory usage by clearing activations of certain layers and recomputing them during a + backward pass. Effectively, this trades extra computation time for reduced memory usage. + cpu_ram_efficient_loading (`bool`, defaults to `None`): + If True, only the first process loads the pretrained model checkpoint while all other processes have empty + weights. Only applicable for Transformers. When using this, `sync_module_states` needs to be `True`. + transformer_cls_names_to_wrap (`Optional[List[str]]`, defaults to `None`): + A list of transformer layer class names to wrap. Only applicable when `auto_wrap_policy` is + `transformer_based_wrap`. + min_num_params (`Optional[int]`, defaults to `None`): + The minimum number of parameters a module must have to be wrapped. Only applicable when `auto_wrap_policy` + is `size_based_wrap`. + """ + + fsdp_version: int = field( + default=None, + metadata={ + "help": "The version of FSDP to use. Defaults to 1. If set to 2, launcher expects the config to be converted to FSDP2 format." + }, + ) + + sharding_strategy: Union[str, "torch.distributed.fsdp.ShardingStrategy"] = field( + default=None, + metadata={ + "help": "Sharding strategy to use. Should be either a `str` or an instance of `torch.distributed.fsdp.fully_sharded_data_parallel.ShardingStrategy`. Defaults to 'FULL_SHARD'. Is deprecated in favor of `reshard_after_forward` " + }, + ) + + reshard_after_forward: Union[str, "torch.distributed.fsdp.ShardingStrategy", bool] = field( + default=None, + metadata={ + "help": "Sharding strategy to use. Should be a bool if `fsdp_version` is set to 2 else a `str` or an instance of `torch.distributed.fsdp.fully_sharded_data_parallel.ShardingStrategy`. Defaults to 'FULL_SHARD'" + }, + ) + backward_prefetch: Optional[Union[str, "torch.distributed.fsdp.BackwardPrefetch"]] = field( + default=None, + metadata={ + "help": "Backward prefetch strategy to use. Should be either a `str` or an instance of `torch.distributed.fsdp.fully_sharded_data_parallel.BackwardPrefetch`. Defaults to 'NO_PREFETCH'. This becomes obsolete in FSDP2." + }, + ) + mixed_precision_policy: Optional[ + Union[ + dict, + str, + "torch.distributed.fsdp.MixedPrecision", + "torch.distributed.fsdp.MixedPrecisionPolicy", + ] + ] = field( + default=None, + metadata={ + "help": "A config to enable mixed precision training with FullyShardedDataParallel. " + "If passing in a `dict`, it should have the following keys: `param_dtype`, `reduce_dtype`, and `buffer_dtype`." + "Can also be an instance of `torch.distributed.fsdp.MixedPrecisionPolicy` if `fsdp_version` is set to 2." + }, + ) + auto_wrap_policy: Optional[Union[Callable, Literal["transformer_based_wrap", "size_based_wrap", "no_wrap"]]] = ( + field( + default=None, + metadata={ + "help": "A callable or string specifying a policy to recursively wrap layers with FSDP. If a string, it must be one of `transformer_based_wrap`, `size_based_wrap`, or `no_wrap`. " + "Defaults to `NO_WRAP`. See `torch.distributed.fsdp.wrap.size_based_wrap_policy` for a direction on what it should look like" + }, + ) + ) + cpu_offload: Union[ + bool, + "torch.distributed.fsdp.CPUOffload", + "torch.distributed.fsdp.CPUOffloadPolicy", + ] = field( + default=None, + metadata={ + "help": "Whether to offload parameters to CPU. Should be either a `bool` or an instance of `torch.distributed.fsdp.fully_sharded_data_parallel.CPUOffload` or `torch.distributed.fsdp.fully_sharded_data_parallel.CPUOffloadPolicy` if `fsdp_version` is set to 2. Defaults to `False`" + }, + ) + ignored_modules: Optional[Union[Iterable[torch.nn.Module], str]] = field( + default=None, + metadata={"help": "A list of modules to ignore when wrapping with FSDP."}, + ) + + state_dict_type: Union[str, "torch.distributed.fsdp.StateDictType"] = field( + default=None, + metadata={ + "help": "State dict type to use. If a string, it must be one of `full_state_dict`, `local_state_dict`, or `sharded_state_dict`. Defaults to `FULL_STATE_DICT`" + }, + ) + state_dict_config: Optional[ + Union[ + "torch.distributed.fsdp.FullStateDictConfig", + "torch.distributed.fsdp.ShardedStateDictConfig", + ] + ] = field( + default=None, + metadata={"help": "State dict config to use. Is determined based on the `state_dict_type` if not passed in."}, + ) + optim_state_dict_config: Optional[ + Union[ + "torch.distributed.fsdp.FullOptimStateDictConfig", + "torch.distributed.fsdp.ShardedOptimStateDictConfig", + ] + ] = field( + default=None, + metadata={ + "help": "Optim state dict config to use. Is determined based on the `state_dict_type` if not passed in." + }, + ) + limit_all_gathers: bool = field( + default=True, + metadata={ + "help": "Whether to have FSDP explicitly synchronizes the CPU thread to prevent " + "too many in-flight all-gathers. This bool only affects the sharded strategies that schedule all-gathers. " + "Enabling this can help lower the number of CUDA malloc retries." + }, + ) + use_orig_params: Optional[bool] = field( + default=None, + metadata={ + "help": "Whether to use the original parameters for the optimizer. Defaults to `False`. This becomes obsolete in FSDP2." + }, + ) + param_init_fn: Optional[Callable[[torch.nn.Module], None]] = field( + default=None, + metadata={ + "help": "A Callable[torch.nn.Module] -> None that specifies how modules " + "that are currently on the meta device should be initialized onto an actual device. " + "Only applicable when `sync_module_states` is `True`. By default is a `lambda` which calls `to_empty` on the module." + }, + ) + sync_module_states: Optional[bool] = field( + default=None, + metadata={ + "help": "Whether each individually wrapped FSDP unit should broadcast module parameters from rank 0 " + "to ensure they are the same across all ranks after initialization. Defaults to `False` unless " + "`cpu_ram_efficient_loading` is `True`, then will be forcibly enabled. This becomes obsolete in FSDP2." + }, + ) + forward_prefetch: bool = field( + default=None, + metadata={ + "help": "Whether to have FSDP explicitly prefetches the next upcoming " + "all-gather while executing in the forward pass. only use with Static graphs. Defaults to `False`" + }, + ) + activation_checkpointing: bool = field( + default=None, + metadata={ + "help": "A technique to reduce memory usage by clearing activations of " + "certain layers and recomputing them during a backward pass. Effectively, this trades extra computation time " + "for reduced memory usage. Defaults to `False`" + }, + ) + cpu_ram_efficient_loading: bool = field( + default=None, + metadata={ + "help": "If True, only the first process loads the pretrained model checkpoint while all other processes have empty weights. " + "Only applicable for 🤗 Transformers. When using this, `sync_module_states` needs to be `True`. Defaults to `False`." + }, + ) + transformer_cls_names_to_wrap: Optional[list[str]] = field( + default=None, + metadata={ + "help": "A list of transformer layer class names to wrap. Only applicable when `auto_wrap_policy` is `transformer_based_wrap`." + }, + ) + min_num_params: Optional[int] = field( + default=None, + metadata={ + "help": "The minimum number of parameters a module must have to be wrapped. Only applicable when `auto_wrap_policy` is `size_based_wrap`." + }, + ) + + def __post_init__(self): + from torch.distributed.fsdp import BackwardPrefetch, ShardingStrategy + + _fsdp2_warnings = set() + + env_prefix = "FSDP_" + # Strategy: By default we should always assume that values are passed in, else we check the environment variables + if self.fsdp_version is None: + self.fsdp_version = int(os.environ.get(env_prefix + "VERSION", "1")) + + if self.fsdp_version == 2: + if not is_torch_version(">=", FSDP2_PYTORCH_VERSION): + raise ImportError(f"FSDP2 requires PyTorch >= {FSDP2_PYTORCH_VERSION}") + + if self.sharding_strategy is not None: + # We cannot properly detect all of the cases, as by default `args.fsdp_sharding_strategy` is set to `fully_shard` + # Therefore we issue a warning only if the user has explicitly set it inside their plugin + _fsdp2_warnings.add( + "sharding_strategy is deprecated in favor of reshard_after_forward. " + "This will be removed in a future version of Accelerate." + ) + if self.fsdp_version == 1: + if self.sharding_strategy is None: + self.sharding_strategy = os.environ.get(env_prefix + "SHARDING_STRATEGY", "FULL_SHARD") + if isinstance(self.sharding_strategy, str): + if self.sharding_strategy.upper() in FSDP_SHARDING_STRATEGY: + self.sharding_strategy = FSDP_SHARDING_STRATEGY.index(self.sharding_strategy.upper()) + 1 + if isinstance(self.sharding_strategy, int) or self.sharding_strategy.isdigit(): + self.sharding_strategy = ShardingStrategy(int(self.sharding_strategy)) + else: + self.sharding_strategy = ShardingStrategy[self.sharding_strategy.upper()] + + # Fallback to `reshard_after_forward` in FSDP1 if `sharding_strategy` is not set + if self.reshard_after_forward is None and self.sharding_strategy is None: + reshard_after_forward = os.environ.get( + env_prefix + "RESHARD_AFTER_FORWARD", + "true" if self.fsdp_version == 2 else "FULL_SHARD", + ) + if self.fsdp_version == 2: + self.reshard_after_forward = str_to_bool(reshard_after_forward.lower(), to_bool=True) + else: + self.reshard_after_forward = reshard_after_forward + if isinstance(self.reshard_after_forward, str): + if self.fsdp_version == 2: + self.reshard_after_forward = str_to_bool(self.reshard_after_forward.lower(), to_bool=True) + else: + # We need to remap based on custom enum values for user readability + if self.reshard_after_forward.upper() in FSDP_SHARDING_STRATEGY: + self.reshard_after_forward = FSDP_SHARDING_STRATEGY.index(self.reshard_after_forward.upper()) + 1 + if isinstance(self.reshard_after_forward, int) or self.reshard_after_forward.isdigit(): + self.reshard_after_forward = ShardingStrategy(int(self.reshard_after_forward)) + else: + self.reshard_after_forward = ShardingStrategy[self.reshard_after_forward.upper()] + + if self.fsdp_version == 2 and not isinstance(self.reshard_after_forward, bool): + raise ValueError( + f"reshard_after_forward set to {self.reshard_after_forward}. This is not supported with FSDP2, please set to a `bool`" + ) + if self.fsdp_version == 1 and isinstance(self.reshard_after_forward, bool): + raise ValueError( + f"reshard_after_forward set to {self.reshard_after_forward}. This is not supported with FSDP1, please set to a `str` or an instance of `torch.distributed.fsdp.fully_sharded_data_parallel.ShardingStrategy`" + ) + + if self.cpu_offload is None: + self.cpu_offload = str_to_bool(os.environ.get(env_prefix + "OFFLOAD_PARAMS", "False")) == 1 + + self.set_cpu_offload() # abstracted away to hide imports due to version checks + self.validate_cpu_offload() + + if self.backward_prefetch is None: + self.backward_prefetch = os.environ.get(env_prefix + "BACKWARD_PREFETCH", None) + if isinstance(self.backward_prefetch, str) and self.backward_prefetch.upper() == "NO_PREFETCH": + self.backward_prefetch = None + if self.backward_prefetch is not None and not isinstance(self.backward_prefetch, BackwardPrefetch): + if isinstance(self.backward_prefetch, str) and self.backward_prefetch.upper() in FSDP_BACKWARD_PREFETCH: + self.backward_prefetch = FSDP_BACKWARD_PREFETCH.index(self.backward_prefetch.upper()) + 1 + if isinstance(self.backward_prefetch, int) or self.backward_prefetch.isdigit(): + self.backward_prefetch = BackwardPrefetch(int(self.backward_prefetch)) + else: + self.backward_prefetch = BackwardPrefetch[self.backward_prefetch.upper()] + if self.fsdp_version == 2 and self.backward_prefetch is not None: + _fsdp2_warnings.add("backward_prefetch is not supported in FSDP2. Setting backward prefetch to None.") + self.backward_prefetch = None + + self.set_state_dict_type() + + if self.auto_wrap_policy is None: + self.auto_wrap_policy = os.environ.get(env_prefix + "AUTO_WRAP_POLICY", "NO_WRAP") + if isinstance(self.auto_wrap_policy, str): + if self.auto_wrap_policy.upper() not in FSDP_AUTO_WRAP_POLICY: + raise ValueError( + f"Invalid auto wrap policy: {self.auto_wrap_policy}. Must be one of {FSDP_AUTO_WRAP_POLICY}" + ) + from torch.distributed.fsdp.wrap import ( + size_based_auto_wrap_policy, + transformer_auto_wrap_policy, + ) + + if self.auto_wrap_policy.upper() == "TRANSFORMER_BASED_WRAP": + self.auto_wrap_policy = transformer_auto_wrap_policy + if self.transformer_cls_names_to_wrap is None: + self.transformer_cls_names_to_wrap = os.environ.get(env_prefix + "TRANSFORMER_CLS_TO_WRAP", None) + if isinstance(self.transformer_cls_names_to_wrap, str): + self.transformer_cls_names_to_wrap = self.transformer_cls_names_to_wrap.split(",") + elif self.auto_wrap_policy.upper() == "SIZE_BASED_WRAP": + self.auto_wrap_policy = size_based_auto_wrap_policy + if self.min_num_params is None: + self.min_num_params = int(os.environ.get(env_prefix + "MIN_NUM_PARAMS", 0)) + elif not isinstance(self.min_num_params, int): + raise ValueError( + f"`min_num_params` must be an integer. Got {self.min_num_params} of type {type(self.min_num_params)}" + ) + elif self.auto_wrap_policy.upper() == "NO_WRAP": + self.auto_wrap_policy = None + + if self.use_orig_params is None and self.fsdp_version == 1: + self.use_orig_params = str_to_bool(os.environ.get(env_prefix + "USE_ORIG_PARAMS", "False")) == 1 + if self.fsdp_version == 2 and self.use_orig_params is not None: + _fsdp2_warnings.add("use_orig_params is obsolete in FSDP2, as FSDP2 always uses the original parameters.") + self.use_orig_params = None + + if self.sync_module_states is None and self.fsdp_version == 1: + self.sync_module_states = str_to_bool(os.environ.get(env_prefix + "SYNC_MODULE_STATES", "False")) == 1 + if self.fsdp_version == 2 and self.sync_module_states is not None: + _fsdp2_warnings.add( + "sync_module_states is obsolete in FSDP2, as it is not needed anymore." + "Setting sync_module_states to None." + ) + self.sync_module_states = None + + if self.forward_prefetch is None and self.fsdp_version == 1: + self.forward_prefetch = str_to_bool(os.environ.get(env_prefix + "FORWARD_PREFETCH", "False")) == 1 + if self.fsdp_version == 2 and self.forward_prefetch is not None: + raise ValueError("forward_prefetch is not yet implemented in FSDP2, set to None or use `fsdp_version=1`") + + if self.activation_checkpointing is None: + self.activation_checkpointing = ( + str_to_bool(os.environ.get(env_prefix + "ACTIVATION_CHECKPOINTING", "False")) == 1 + ) + + if self.ignored_modules is None: + self.ignored_modules = os.environ.get(env_prefix + "IGNORED_MODULES", None) + + if self.cpu_ram_efficient_loading is None: + self.cpu_ram_efficient_loading = ( + str_to_bool(os.environ.get(env_prefix + "CPU_RAM_EFFICIENT_LOADING", "False")) == 1 + ) + else: + # We still need to set it for transformers + os.environ[env_prefix + "CPU_RAM_EFFICIENT_LOADING"] = str(self.cpu_ram_efficient_loading) + # There's no need to specify sync_module_states in FSDP2 + if self.fsdp_version == 1 and self.cpu_ram_efficient_loading and not self.sync_module_states: + warnings.warn( + "sync_module_states cannot be False since efficient cpu ram loading enabled. " + "Setting sync_module_states to True." + ) + self.sync_module_states = True + if isinstance(self.mixed_precision_policy, str): + # override is True since self.mixed_precision_policy is not None + # has to be overwritten with the correct mixed precision object + self.set_mixed_precision(self.mixed_precision_policy, override=True) + elif isinstance(self.mixed_precision_policy, dict): + self.set_mixed_precision(self.mixed_precision_policy) + if self.mixed_precision_policy is not None: + self.validate_mixed_precision_policy() + + if self.sync_module_states: + if is_npu_available(): + device = torch.npu.current_device() + elif is_mlu_available(): + device = torch.mlu.current_device() + elif is_musa_available(): + device = torch.musa.current_device() + elif is_cuda_available(): + device = torch.cuda.current_device() + elif is_xpu_available(): + device = torch.xpu.current_device() + elif is_hpu_available(): + device = torch.hpu.current_device() + elif is_neuron_available(): + device = torch.neuron.current_device() + else: + raise RuntimeError( + "There are currently no available devices found, must be one of 'XPU', 'CUDA', 'MLU', 'NPU', 'MUSA', or 'HPU'." + ) + # Create a function that will be used to initialize the parameters of the model + # when using `sync_module_states` + self.param_init_fn = lambda x: x.to_empty(device=device, recurse=False) + if is_torch_version("<", "2.7.0") and self.fsdp_version == 2 and self.ignored_modules is not None: + _fsdp2_warnings.add( + "FSDP2 ignored_params/ignored_modules is not available for torch version < 2.7.0" + "Setting ignored_modules to None." + ) + self.ignored_modules = None + # Single warning for all deprecation warnings due to FSDP2 conversion + if _fsdp2_warnings: + logger.warning("Multiple deprecation warnings due to FSDP2 conversion:\n".join(_fsdp2_warnings)) + + def set_state_dict_type(self, state_dict_type=None): + """ + Set the state dict config based on the `StateDictType`. + """ + from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, + FullStateDictConfig, + ShardedOptimStateDictConfig, + ShardedStateDictConfig, + StateDictType, + ) + + # Override the state_dict_type if provided, typical use case: + # user trains with sharded, but final save is with full + if state_dict_type is not None: + self.state_dict_type = state_dict_type + + if self.state_dict_type is None: + self.state_dict_type = os.environ.get( + "FSDP_STATE_DICT_TYPE", + "FULL_STATE_DICT" if self.fsdp_version == 1 else "SHARDED_STATE_DICT", + ) + if isinstance(self.state_dict_type, str): + if self.state_dict_type.isdigit(): + self.state_dict_type = StateDictType(int(self.state_dict_type)) + else: + self.state_dict_type = StateDictType[self.state_dict_type.upper()] + + if self.state_dict_type == StateDictType.FULL_STATE_DICT: + if self.state_dict_config is None: + self.state_dict_config = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + if self.optim_state_dict_config is None: + self.optim_state_dict_config = FullOptimStateDictConfig(offload_to_cpu=True, rank0_only=True) + elif self.state_dict_type == StateDictType.SHARDED_STATE_DICT: + if self.state_dict_config is None: + self.state_dict_config = ShardedStateDictConfig(offload_to_cpu=True) + if self.optim_state_dict_config is None: + self.optim_state_dict_config = ShardedOptimStateDictConfig(offload_to_cpu=True) + + if self.fsdp_version == 2 and self.state_dict_type == StateDictType.LOCAL_STATE_DICT: + raise ValueError( + "FSDP2 does not support LOCAL_STATE_DICT. " + "Please set `fsdp_state_dict_type` to `SHARDED_STATE_DICT` or `FULL_STATE_DICT`." + ) + + def set_auto_wrap_policy(self, model): + """ + Given `model`, creates an `auto_wrap_policy` based on the passed in policy and if we can use the + `transformer_cls_to_wrap` + """ + from torch.distributed.fsdp.wrap import ( + size_based_auto_wrap_policy, + transformer_auto_wrap_policy, + ) + + # First base off of `_no_split_modules` + no_split_modules = getattr(model, "_no_split_modules", None) + default_transformer_cls_names_to_wrap = list(no_split_modules) if no_split_modules is not None else [] + if self.auto_wrap_policy == transformer_auto_wrap_policy: + if self.transformer_cls_names_to_wrap is None: + self.transformer_cls_names_to_wrap = default_transformer_cls_names_to_wrap + transformer_cls_to_wrap = set() + for layer_class in self.transformer_cls_names_to_wrap: + transformer_cls = get_module_class_from_name(model, layer_class) + if transformer_cls is None: + raise ValueError(f"Could not find the transformer layer class {layer_class} in the model.") + transformer_cls_to_wrap.add(transformer_cls) + # Finally we set the auto_wrap_policy to a callable + self.auto_wrap_policy = functools.partial( + self.auto_wrap_policy, transformer_layer_cls=transformer_cls_to_wrap + ) + + elif self.auto_wrap_policy == size_based_auto_wrap_policy: + # If zero, we silently ignore it. + if self.min_num_params > 0: + self.auto_wrap_policy = functools.partial(self.auto_wrap_policy, min_num_params=self.min_num_params) + else: + self.auto_wrap_policy = None + + def set_mixed_precision(self, mixed_precision, buffer_autocast=False, override=False): + "Sets the mixed precision policy for FSDP" + mixed_precision_mapping = { + "fp8": torch.bfloat16, + "fp16": torch.float16, + "bf16": torch.bfloat16, + "fp32": torch.float32, + } + dtype = mixed_precision + if isinstance(mixed_precision, str): + dtype = mixed_precision_mapping.get(mixed_precision, None) + if dtype is None: + raise ValueError( + f"Invalid mixed precision: {mixed_precision}. Must be one of {list(mixed_precision_mapping.keys())}" + ) + elif isinstance(mixed_precision, torch.dtype) and mixed_precision not in mixed_precision_mapping.values(): + raise ValueError( + f"Invalid mixed precision: {mixed_precision}. Must be one of {list(mixed_precision_mapping.values())}" + ) + + buffer_type = torch.float32 if buffer_autocast else dtype + + if self.fsdp_version == 1: + from torch.distributed.fsdp import MixedPrecision + elif self.fsdp_version == 2: + from torch.distributed.fsdp import MixedPrecisionPolicy as MixedPrecision + + if override or self.mixed_precision_policy is None: + dtype_args = {"param_dtype": dtype, "reduce_dtype": dtype} + if self.fsdp_version == 1: + dtype_args["buffer_dtype"] = buffer_type + else: + dtype_args["output_dtype"] = dtype + # TODO(s1ro1): `cast_forward_inputs` for FSDP2? + self.mixed_precision_policy = MixedPrecision(**dtype_args) + elif isinstance(self.mixed_precision_policy, dict): + # Check for incompatible types + valid_keys = ["param_dtype", "reduce_dtype"] + ( + ["buffer_dtype"] if self.fsdp_version == 1 else ["output_dtype"] + ) + missing_keys = [k for k in valid_keys if k not in self.mixed_precision_policy] + invalid_values = [ + k for k, v in self.mixed_precision_policy.items() if v not in mixed_precision_mapping.values() + ] + if missing_keys or invalid_values: + raise ValueError( + f"Invalid mixed precision policy: {self.mixed_precision_policy}. " + f"Must be a `dict` with keys {valid_keys}." + f"Values must be one of {list(mixed_precision_mapping.values())}" + ) + self.mixed_precision_policy = MixedPrecision(**self.mixed_precision_policy) + + def validate_mixed_precision_policy(self): + """ + Validates the mixed precision policy, abstracted away to not bring in the imports if not needed. + """ + if self.fsdp_version == 2: + from torch.distributed.fsdp import MixedPrecisionPolicy as MixedPrecision + else: + from torch.distributed.fsdp import MixedPrecision + + if not isinstance(self.mixed_precision_policy, MixedPrecision): + required_type = ( + "`torch.distributed.fsdp.MixedPrecisionPolicy`" + if self.fsdp_version == 2 + else "`torch.distributed.fsdp.MixedPrecision`" + ) + raise ValueError(f"mixed_precision_policy must be an instance of {required_type}.") + + def set_cpu_offload(self): + if self.fsdp_version == 2: + from torch.distributed.fsdp import CPUOffloadPolicy, OffloadPolicy + else: + from torch.distributed.fsdp import CPUOffload + + if isinstance(self.cpu_offload, bool): + if self.fsdp_version == 2: + if not self.cpu_offload: + self.cpu_offload = OffloadPolicy() + else: + self.cpu_offload = CPUOffloadPolicy() + else: + self.cpu_offload = CPUOffload(offload_params=self.cpu_offload) + + def validate_cpu_offload(self): + if self.fsdp_version == 2: + from torch.distributed.fsdp import OffloadPolicy + else: + from torch.distributed.fsdp import CPUOffload + + if self.fsdp_version == 2 and not isinstance(self.cpu_offload, OffloadPolicy): + raise ValueError( + f"`cpu_offload` must be an instance of `torch.distributed.fsdp.OffloadPolicy` in FSDP2, got {self.cpu_offload}" + ) + if self.fsdp_version == 1 and not isinstance(self.cpu_offload, CPUOffload): + raise ValueError( + f"`cpu_offload` must be an instance of `torch.distributed.fsdp.CPUOffload` in FSDP1, got {self.cpu_offload}" + ) + + +@dataclass +class TorchTensorParallelPlugin: + """ + This plugin is used to enable tensor parallelism using PyTorch >= 2.0. + """ + + tp_size: int = field( + default=1, + metadata={"help": "tensor parallel size will be used in the device mesh preparation"}, + ) + + # torch_device_mesh is of type "torch.distributed.DeviceMesh" + torch_device_mesh: Optional["torch.distributed.DeviceMesh"] = field(default=None) + + +@dataclass +class TorchContextParallelConfig: + """ + This class holds the configuration for context parallelism in PyTorch. + """ + + cp_comm_strategy: Optional[str] = field( + default=None, + metadata={ + "help": "Communication strategy for context parallelism. Can be one of 'allgather' or 'alltoall'. Defaults to 'allgather'." + }, + ) + + def __post_init__(self): + if not is_torch_version(">=", BETA_CP_AVAILABLE_PYTORCH_VERSION): + raise ValueError( + f"FSDP2-based Context parallelism is only available in PyTorch {BETA_CP_AVAILABLE_PYTORCH_VERSION} and later versions. " + "Please upgrade your PyTorch version." + ) + + if self.cp_comm_strategy is None: + self.cp_comm_strategy = os.environ.get("PARALLELISM_CONFIG_CP_COMM_STRATEGY", "allgather") + if self.cp_comm_strategy not in ["allgather", "alltoall"]: + raise ValueError( + f"Invalid cp_comm_strategy: {self.cp_comm_strategy}. Must be one of 'allgather' or 'alltoall'." + ) + + +@dataclass +class DeepSpeedSequenceParallelConfig: + sp_seq_length: Optional[int] = field( + default=None, + metadata={ + "help": "Sequence length for when batches are all of the same length. For variable sequence lengths across batches set `sp_seq_length_is_variable=True` and leave this field unset" + }, + ) + sp_seq_length_is_variable: Optional[bool] = field( + default=None, + metadata={ + "help": "If `True` will work with a sequence length that may change between batches, in which case `sp_seq_length` value can be set to anything divisible by cp size or remain unset. If `False` then `sp_seq_length` needs to match the batch's sequence length dimension. The default is `True`." + }, + ) + sp_attn_implementation: Optional[str] = field( + default=None, + metadata={ + "help": "Attention implementation to use. Can be one of 'flash_attention_2', 'flash_attention_3', 'sdpa', or a hub-hosted kernel (e.g. 'kernels-community/flash-attn2'). Defaults to `sdpa`." + }, + ) + + def __post_init__(self): + # sp_seq_length_is_variable and sp_seq_length are interconnected + if self.sp_seq_length_is_variable is None: + self.sp_seq_length_is_variable = ( + os.environ.get("PARALLELISM_CONFIG_SP_SEQ_LENGTH_IS_VARIABLE", "true").lower() == "true" + ) + + if not self.sp_seq_length_is_variable and self.sp_seq_length is None: + if "PARALLELISM_CONFIG_SP_SEQ_LENGTH" not in os.environ: + raise ValueError( + "when `sp_seq_length_is_variable` is `False` `sp_seq_length` must be provided either through the constructor or the environment variable PARALLELISM_CONFIG_SP_SEQ_LENGTH" + ) + else: + self.sp_seq_length = os.environ.get("PARALLELISM_CONFIG_SP_SEQ_LENGTH") + self.sp_seq_length = None if self.sp_seq_length == "None" else int(self.sp_seq_length) + + if self.sp_attn_implementation is None: + self.sp_attn_implementation = os.environ.get("PARALLELISM_CONFIG_SP_ATTN_IMPLEMENTATION", None) + + _builtin_sp_attn = ["flash_attention_2", "flash_attention_3", "sdpa"] + # Also allow hub-hosted flash attention kernels (e.g. "kernels-community/flash-attn2"). + # These register into transformers' ALL_ATTENTION_FUNCTIONS at model load time and + # DeepSpeed validates against that registry directly. + _unsupported_sp_attn = ["eager", "flex_attention"] + if self.sp_attn_implementation is not None: + if self.sp_attn_implementation in _unsupported_sp_attn: + raise ValueError( + f"Invalid sp_attn_implementation: {self.sp_attn_implementation}. " + f"'eager' and 'flex_attention' are not supported with sequence parallelism." + ) + if self.sp_attn_implementation not in _builtin_sp_attn: + if "/" not in self.sp_attn_implementation or "flash-attn" not in self.sp_attn_implementation: + raise ValueError( + f"Invalid sp_attn_implementation: {self.sp_attn_implementation}. " + f"Must be one of {_builtin_sp_attn} or a hub-hosted flash attention kernel " + f"(e.g. 'kernels-community/flash-attn2')." + ) + + +@dataclass +class TorchTensorParallelConfig: + """ + Use this object in your [`Accelerator`] to customize your torch tensor parallelism. + """ + + enable_async_tp: bool = False + + def __post_init__(self): + if not is_torch_version(">=", BETA_TP_AVAILABLE_PYTORCH_VERSION): + raise ValueError( + f"Torch tensor parallelism is only available in PyTorch {BETA_TP_AVAILABLE_PYTORCH_VERSION} and later versions. " + "Please upgrade your PyTorch version." + ) + + if not compare_versions("transformers", ">=", BETA_TP_AVAILABLE_TRANSFORMERS_VERSION): + raise ValueError(f"TP requires transformers >= {BETA_TP_AVAILABLE_TRANSFORMERS_VERSION}") + + if self.enable_async_tp: + warnings.warn("Async tensor parallelism is currently not supported, ignoring this option.") + + +@dataclass +class MegatronLMPlugin: + """ + Plugin for Megatron-LM to enable tensor, pipeline, sequence and data parallelism. Also to enable selective + activation recomputation and optimized fused kernels. + + Args: + tp_degree (`int`, defaults to `None`): + Tensor parallelism degree. + pp_degree (`int`, defaults to `None`): + Pipeline parallelism degree. + num_micro_batches (`int`, defaults to `None`): + Number of micro-batches. + gradient_clipping (`float`, defaults to `None`): + Gradient clipping value based on global L2 Norm (0 to disable). + sequence_parallelism (`bool`, defaults to `None`): + Enable sequence parallelism. + recompute_activations (`bool`, defaults to `None`): + Enable selective activation recomputation. + use_distributed_optimizr (`bool`, defaults to `None`): + Enable distributed optimizer. + pipeline_model_parallel_split_rank (`int`, defaults to `None`): + Rank where encoder and decoder should be split. + num_layers_per_virtual_pipeline_stage (`int`, defaults to `None`): + Number of layers per virtual pipeline stage. + is_train_batch_min (`str`, defaults to `True`): + If both tran & eval dataloaders are specified, this will decide the `micro_batch_size`. + train_iters (`int`, defaults to `None`): + Total number of samples to train over all training runs. Note that either train-iters or train-samples + should be provided when using `MegatronLMDummyScheduler`. + train_samples (`int`, defaults to `None`): + Total number of samples to train over all training runs. Note that either train-iters or train-samples + should be provided when using `MegatronLMDummyScheduler`. + weight_decay_incr_style (`str`, defaults to `'constant'`): + Weight decay increment function. choices=["constant", "linear", "cosine"]. + start_weight_decay (`float`, defaults to `None`): + Initial weight decay coefficient for L2 regularization. + end_weight_decay (`float`, defaults to `None`): + End of run weight decay coefficient for L2 regularization. + lr_decay_style (`str`, defaults to `'linear'`): + Learning rate decay function. choices=['constant', 'linear', 'cosine']. + lr_decay_iters (`int`, defaults to `None`): + Number of iterations for learning rate decay. If None defaults to `train_iters`. + lr_decay_samples (`int`, defaults to `None`): + Number of samples for learning rate decay. If None defaults to `train_samples`. + lr_warmup_iters (`int`, defaults to `None`): + Number of iterations to linearly warmup learning rate over. + lr_warmup_samples (`int`, defaults to `None`): + Number of samples to linearly warmup learning rate over. + lr_warmup_fraction (`float`, defaults to `None`): + Fraction of lr-warmup-(iters/samples) to linearly warmup learning rate over. + min_lr (`float`, defaults to `0`): + Minimum value for learning rate. The scheduler clip values below this threshold. + consumed_samples (`List`, defaults to `None`): + Number of samples consumed in the same order as the dataloaders to `accelerator.prepare` call. + no_wd_decay_cond (`Optional`, defaults to `None`): + Condition to disable weight decay. + scale_lr_cond (`Optional`, defaults to `None`): + Condition to scale learning rate. + lr_mult (`float`, defaults to `1.0`): + Learning rate multiplier. + megatron_dataset_flag (`bool`, defaults to `False`): + Whether the format of dataset follows Megatron-LM Indexed/Cached/MemoryMapped format. + seq_length (`int`, defaults to `None`): + Maximum sequence length to process. + encoder_seq_length (`int`, defaults to `None`): + Maximum sequence length to process for the encoder. + decoder_seq_length (`int`, defaults to `None`): + Maximum sequence length to process for the decoder. + tensorboard_dir (`str`, defaults to `None`): + Path to save tensorboard logs. + set_all_logging_options (`bool`, defaults to `False`): + Whether to set all logging options. + eval_iters (`int`, defaults to `100`): + Number of iterations to run for evaluation validation/test for. + eval_interval (`int`, defaults to `1000`): + Interval between running evaluation on validation set. + return_logits (`bool`, defaults to `False`): + Whether to return logits from the model. + custom_train_step_class (`Optional`, defaults to `None`): + Custom train step class. + custom_train_step_kwargs (`Optional`, defaults to `None`): + Custom train step kwargs. + custom_model_provider_function (`Optional`, defaults to `None`): + Custom model provider function. + custom_prepare_model_function (`Optional`, defaults to `None`): + Custom prepare model function. + custom_megatron_datasets_provider_function (`Optional`, defaults to `None`): + Custom megatron train_valid_test datasets provider function. + custom_get_batch_function (`Optional`, defaults to `None`): + Custom get batch function. + custom_loss_function (`Optional`, defaults to `None`): + Custom loss function. + other_megatron_args (`Optional`, defaults to `None`): + Other Megatron-LM arguments. Please refer Megatron-LM. + """ + + tp_degree: int = field(default=None, metadata={"help": "tensor parallelism degree."}) + pp_degree: int = field(default=None, metadata={"help": "pipeline parallelism degree."}) + use_custom_fsdp: bool = field(default=None, metadata={"help": "use custom fsdp."}) + overlap_cpu_optimizer_d2h_h2d: bool = field( + default=None, metadata={"help": "overlap CPU optimizer step, gradients D2H and updated parameters H2D."} + ) + no_load_optim: bool = field(default=None, metadata={"help": "do not load optimizer."}) + eod_mask_loss: bool = field(default=None, metadata={"help": "use eod mask loss."}) + no_save_optim: bool = field(default=None, metadata={"help": "do not save optimizer."}) + optimizer_cpu_offload: bool = field(default=None, metadata={"help": "use CPU offload for optimizer."}) + use_precision_aware_optimizer: bool = field(default=None, metadata={"help": "use precision aware optimizer."}) + decoder_last_pipeline_num_layers: int = field( + default=None, + metadata={ + "help": "decoder last pipeline number of layers, default None is even split of transformer layers across all pipeline stages." + }, + ) + recompute_granularity: str = field(default=None, metadata={"help": "recompute granularity (full, selective)."}) + recompute_method: str = field(default=None, metadata={"help": "recompute method (uniform, block)."}) + recompute_num_layers: int = field(default=None, metadata={"help": "number of layers to recompute."}) + attention_backend: bool = field(default=None, metadata={"help": "enable attention backend."}) + expert_model_parallel_size: int = field(default=None, metadata={"help": "expert model parallel size."}) + context_parallel_size: int = field(default=None, metadata={"help": "context parallel size."}) + attention_dropout: float = field(default=None, metadata={"help": "attention dropout rate."}) + hidden_dropout: float = field(default=None, metadata={"help": "hidden dropout rate."}) + attention_softmax_in_fp32: bool = field(default=None, metadata={"help": "use fp32 for attention softmax."}) + expert_tensor_parallel_size: int = field(default=None, metadata={"help": "expert tensor parallel size."}) + calculate_per_token_loss: bool = field(default=None, metadata={"help": "calculate per token loss."}) + use_rotary_position_embeddings: bool = field(default=None, metadata={"help": "use rotary position embeddings."}) + num_micro_batches: int = field(default=None, metadata={"help": "number of micro-batches."}) + gradient_clipping: float = field( + default=None, + metadata={"help": "gradient clipping value based on global L2 Norm (0 to disable)"}, + ) + sequence_parallelism: bool = field( + default=None, + metadata={"help": "enable sequence parallelism"}, + ) + recompute_activations: bool = field( + default=None, + metadata={"help": "enable selective activation recomputation"}, + ) + use_distributed_optimizer: bool = field( + default=None, + metadata={"help": "enable distributed optimizer"}, + ) + pipeline_model_parallel_split_rank: int = field( + default=None, + metadata={"help": "Rank where encoder and decoder should be split."}, + ) + num_layers_per_virtual_pipeline_stage: int = field( + default=None, metadata={"help": "Number of layers per virtual pipeline stage."} + ) + is_train_batch_min: str = field( + default=True, + metadata={"help": "If both train & eval dataloaders are specified, this will decide the micro_batch_size"}, + ) + train_iters: int = field( + default=None, + metadata={ + "help": "Total number of iterations to train over all training runs. " + "Note that either train-iters or train-samples should be provided when using `MegatronLMDummyScheduler`" + }, + ) + train_samples: int = field( + default=None, + metadata={ + "help": "Total number of samples to train over all training runs. " + "Note that either train-iters or train-samples should be provided when using `MegatronLMDummyScheduler`" + }, + ) + weight_decay_incr_style: str = field( + default="constant", + metadata={"help": 'Weight decay increment function. choices=["constant", "linear", "cosine"]. '}, + ) + start_weight_decay: float = field( + default=None, + metadata={"help": "Initial weight decay coefficient for L2 regularization."}, + ) + end_weight_decay: float = field( + default=None, + metadata={"help": "End of run weight decay coefficient for L2 regularization."}, + ) + lr_decay_style: str = field( + default="linear", + metadata={"help": "Learning rate decay function. choices=['constant', 'linear', 'cosine']."}, + ) + lr_decay_iters: int = field( + default=None, + metadata={"help": "Number of iterations for learning rate decay. If None defaults to `train_iters`."}, + ) + lr_decay_samples: int = field( + default=None, + metadata={"help": "Number of samples for learning rate decay. If None defaults to `train_samples`."}, + ) + lr_warmup_iters: int = field( + default=None, + metadata={"help": "number of iterations to linearly warmup learning rate over."}, + ) + lr_warmup_samples: int = field( + default=None, + metadata={"help": "number of samples to linearly warmup learning rate over."}, + ) + lr_warmup_fraction: float = field( + default=None, + metadata={"help": "fraction of lr-warmup-(iters/samples) to linearly warmup learning rate over."}, + ) + min_lr: float = field( + default=0, + metadata={"help": "Minimum value for learning rate. The scheduler clip values below this threshold."}, + ) + consumed_samples: list[int] = field( + default=None, + metadata={ + "help": "Number of samples consumed in the same order as the dataloaders to `accelerator.prepare` call." + }, + ) + no_wd_decay_cond: Optional[Callable] = field(default=None, metadata={"help": "Condition to disable weight decay."}) + scale_lr_cond: Optional[Callable] = field(default=None, metadata={"help": "Condition to scale learning rate."}) + lr_mult: float = field(default=1.0, metadata={"help": "Learning rate multiplier."}) + megatron_dataset_flag: bool = field( + default=False, + metadata={"help": "Whether the format of dataset follows Megatron-LM Indexed/Cached/MemoryMapped format."}, + ) + seq_length: int = field( + default=None, + metadata={"help": "Maximum sequence length to process."}, + ) + encoder_seq_length: int = field( + default=None, + metadata={"help": "Maximum sequence length to process for the encoder."}, + ) + decoder_seq_length: int = field( + default=None, + metadata={"help": "Maximum sequence length to process for the decoder."}, + ) + tensorboard_dir: str = field( + default=None, + metadata={"help": "Path to save tensorboard logs."}, + ) + set_all_logging_options: bool = field( + default=False, + metadata={"help": "Whether to set all logging options."}, + ) + eval_iters: int = field( + default=100, + metadata={"help": "Number of iterations to run for evaluation validation/test for."}, + ) + eval_interval: int = field( + default=1000, + metadata={"help": "Interval between running evaluation on validation set."}, + ) + return_logits: bool = field( + default=False, + metadata={"help": "Whether to return logits from the model."}, + ) + + # custom train step args + custom_train_step_class: Optional[Any] = field( + default=None, + metadata={"help": "Custom train step class."}, + ) + custom_train_step_kwargs: Optional[dict[str, Any]] = field( + default=None, + metadata={"help": "Custom train step kwargs."}, + ) + + # custom model args + custom_model_provider_function: Optional[Callable] = field( + default=None, + metadata={"help": "Custom model provider function."}, + ) + custom_prepare_model_function: Optional[Callable] = field( + default=None, + metadata={"help": "Custom prepare model function."}, + ) + custom_megatron_datasets_provider_function: Optional[Callable] = field( + default=None, + metadata={"help": "Custom megatron train_valid_test datasets provider function."}, + ) + custom_get_batch_function: Optional[Callable] = field( + default=None, + metadata={"help": "Custom get batch function."}, + ) + custom_loss_function: Optional[Callable] = field( + default=None, + metadata={"help": "Custom loss function."}, + ) + + # remaining args such as enabling Alibi/ROPE positional embeddings, + # wandb logging, Multi-Query Attention, etc. + other_megatron_args: Optional[dict[str, Any]] = field( + default=None, + metadata={"help": "Other Megatron-LM arguments. Please refer Megatron-LM"}, + ) + + def __post_init__(self): + prefix = "MEGATRON_LM_" + if self.tp_degree is None: + self.tp_degree = int(os.environ.get(prefix + "TP_DEGREE", 1)) + if self.pp_degree is None: + self.pp_degree = int(os.environ.get(prefix + "PP_DEGREE", 1)) + if self.use_custom_fsdp is None: + self.use_custom_fsdp = str_to_bool(os.environ.get(prefix + "USE_CUSTOM_FSDP", "False")) == 1 + if self.no_load_optim is None: + self.no_load_optim = str_to_bool(os.environ.get(prefix + "NO_LOAD_OPTIM", "False")) == 1 + if self.eod_mask_loss is None: + self.eod_mask_loss = str_to_bool(os.environ.get(prefix + "EOD_MASK_LOSS", "False")) == 1 + if self.no_save_optim is None: + self.no_save_optim = str_to_bool(os.environ.get(prefix + "NO_SAVE_OPTIM", "False")) == 1 + if self.optimizer_cpu_offload is None: + self.optimizer_cpu_offload = str_to_bool(os.environ.get(prefix + "OPTIMIZER_CPU_OFFLOAD", "False")) == 1 + if self.overlap_cpu_optimizer_d2h_h2d is None: + self.overlap_cpu_optimizer_d2h_h2d = ( + str_to_bool(os.environ.get(prefix + "OVERLAP_CPU_OPTIMIZER_D2H_H2D", "False")) == 1 + ) + if self.use_precision_aware_optimizer is None: + self.use_precision_aware_optimizer = ( + str_to_bool(os.environ.get(prefix + "USE_PRECISION_AWARE_OPTIMIZER", "False")) == 1 + ) + if self.decoder_last_pipeline_num_layers is None: + if os.environ.get(prefix + "DECODER_LAST_PIPELINE_NUM_LAYERS") is not None: + self.decoder_last_pipeline_num_layers = int( + os.environ.get(prefix + "DECODER_LAST_PIPELINE_NUM_LAYERS", 0) + ) + else: + self.decoder_last_pipeline_num_layers = None + if self.num_micro_batches is None: + self.num_micro_batches = int(os.environ.get(prefix + "NUM_MICRO_BATCHES", 1)) + if self.gradient_clipping is None: + self.gradient_clipping = float(os.environ.get(prefix + "GRADIENT_CLIPPING", 1.0)) + if self.recompute_activations is None: + self.recompute_activations = str_to_bool(os.environ.get(prefix + "RECOMPUTE_ACTIVATIONS", "False")) == 1 + if self.use_distributed_optimizer is None: + self.use_distributed_optimizer = ( + str_to_bool(os.environ.get(prefix + "USE_DISTRIBUTED_OPTIMIZER", "False")) == 1 + ) + if self.sequence_parallelism is None: + self.sequence_parallelism = str_to_bool(os.environ.get(prefix + "SEQUENCE_PARALLELISM", "False")) == 1 + if self.recompute_granularity is None: + self.recompute_granularity = os.environ.get(prefix + "RECOMPUTE_GRANULARITY", "full") + if self.recompute_method is None: + self.recompute_method = os.environ.get(prefix + "RECOMPUTE_METHOD", "uniform") + if self.recompute_num_layers is None: + self.recompute_num_layers = int(os.environ.get(prefix + "RECOMPUTE_NUM_LAYERS", 1)) + if self.attention_backend is None: + self.attention_backend = str_to_bool(os.environ.get(prefix + "ATTENTION_BACKEND", "True")) == 1 + if self.expert_model_parallel_size is None: + self.expert_model_parallel_size = int(os.environ.get(prefix + "EXPERT_MODEL_PARALLEL_SIZE", 1)) + if self.context_parallel_size is None: + self.context_parallel_size = int(os.environ.get(prefix + "CONTEXT_PARALLEL_SIZE", 2)) + if self.attention_dropout is None: + self.attention_dropout = float(os.environ.get(prefix + "ATTENTION_DROPOUT", "0.0")) + if self.hidden_dropout is None: + self.hidden_dropout = float(os.environ.get(prefix + "HIDDEN_DROPOUT", "0.0")) + if self.attention_softmax_in_fp32 is None: + self.attention_softmax_in_fp32 = ( + str_to_bool(os.environ.get(prefix + "ATTENTION_SOFTMAX_IN_FP32", "True")) == 1 + ) + if self.expert_tensor_parallel_size is None: + self.expert_tensor_parallel_size = int(os.environ.get(prefix + "EXPERT_TENSOR_PARALLEL_SIZE", 1)) + if self.calculate_per_token_loss is None: + self.calculate_per_token_loss = ( + str_to_bool(os.environ.get(prefix + "CALCULATE_PER_TOKEN_LOSS", "True")) == 1 + ) + if self.use_rotary_position_embeddings is None: + self.use_rotary_position_embeddings = ( + str_to_bool(os.environ.get(prefix + "USE_ROTARY_POSITION_EMBEDDINGS", "True")) == 1 + ) + + if self.pp_degree > 1 or self.use_distributed_optimizer: + self.DDP_impl = "local" + else: + self.DDP_impl = "torch" + + if self.consumed_samples is not None: + if len(self.consumed_samples) == 1: + self.consumed_samples.extend([0, 0]) + elif len(self.consumed_samples) == 2: + self.consumed_samples.append(0) + + self.megatron_lm_default_args = { + "tensor_model_parallel_size": self.tp_degree, + "pipeline_model_parallel_size": self.pp_degree, + "pipeline_model_parallel_split_rank": self.pipeline_model_parallel_split_rank, + "num_layers_per_virtual_pipeline_stage": self.num_layers_per_virtual_pipeline_stage, + "DDP_impl": self.DDP_impl, + "use_distributed_optimizer": self.use_distributed_optimizer, + "sequence_parallel": self.sequence_parallelism, + "clip_grad": self.gradient_clipping, + "num_micro_batches": self.num_micro_batches, + "consumed_samples": self.consumed_samples, + "no_wd_decay_cond": self.no_wd_decay_cond, + "scale_lr_cond": self.scale_lr_cond, + "lr_mult": self.lr_mult, + "megatron_dataset_flag": self.megatron_dataset_flag, + "eval_iters": self.eval_iters, + "eval_interval": self.eval_interval, + "use_custom_fsdp": self.use_custom_fsdp, + "no_load_optim": self.no_load_optim, + "eod_mask_loss": self.eod_mask_loss, + "no_save_optim": self.no_save_optim, + "optimizer_cpu_offload": self.optimizer_cpu_offload, + "overlap_cpu_optimizer_d2h_h2d": self.overlap_cpu_optimizer_d2h_h2d, + "use_precision_aware_optimizer": self.use_precision_aware_optimizer, + "decoder_last_pipeline_num_layers": self.decoder_last_pipeline_num_layers, + "recompute_granularity": self.recompute_granularity, + "recompute_method": self.recompute_method, + "recompute_num_layers": self.recompute_num_layers, + "attention_backend": self.attention_backend, + "expert_model_parallel_size": self.expert_model_parallel_size, + "context_parallel_size": self.context_parallel_size, + "attention_dropout": self.attention_dropout, + "hidden_dropout": self.hidden_dropout, + "attention_softmax_in_fp32": self.attention_softmax_in_fp32, + "expert_tensor_parallel_size": self.expert_tensor_parallel_size, + "calculate_per_token_loss": self.calculate_per_token_loss, + "use_rotary_position_embeddings": self.use_rotary_position_embeddings, + } + if self.tensorboard_dir is not None: + self.megatron_lm_default_args["tensorboard_dir"] = self.tensorboard_dir + if self.set_all_logging_options: + self.set_tensorboard_logging_options() + if self.other_megatron_args is not None: + self.megatron_lm_default_args.update(self.other_megatron_args) + + def set_network_size_args(self, model, batch_data=None): + model_config_type = model.config.model_type.lower() + for model_type in MODEL_CONFIGS_TO_MEGATRON_PARSERS.keys(): + if model_type in model_config_type: + MODEL_CONFIGS_TO_MEGATRON_PARSERS[model_type](self, model, batch_data) + return + raise ValueError( + f"Accelerate Megatron-LM integration not supports {model_config_type} model. " + "You can add your own model config parser." + ) + + def set_mixed_precision(self, mixed_precision): + if mixed_precision == "fp16": + self.megatron_lm_default_args["fp16"] = True + elif mixed_precision == "bf16": + self.megatron_lm_default_args["bf16"] = True + self.DDP_impl = "local" + self.megatron_lm_default_args["DDP_impl"] = self.DDP_impl + + def set_training_args(self, micro_batch_size, dp_degree): + self.data_parallel_size = dp_degree + self.micro_batch_size = micro_batch_size + self.global_batch_size = dp_degree * micro_batch_size * self.num_micro_batches + self.megatron_lm_default_args["data_parallel_size"] = self.data_parallel_size + self.megatron_lm_default_args["micro_batch_size"] = self.micro_batch_size + self.megatron_lm_default_args["global_batch_size"] = self.global_batch_size + + def set_optimizer_type(self, optimizer): + optimizer_name = optimizer.__class__.__name__.lower() + if "adam" in optimizer_name: + self.megatron_lm_default_args["optimizer"] = "adam" + self.megatron_lm_default_args["adam_beta1"] = optimizer.defaults["betas"][0] + self.megatron_lm_default_args["adam_beta2"] = optimizer.defaults["betas"][1] + self.megatron_lm_default_args["adam_eps"] = optimizer.defaults["eps"] + elif "sgd" in optimizer_name: + self.megatron_lm_default_args["optimizer"] = "sgd" + self.megatron_lm_default_args["sgd_momentum"] = optimizer.defaults["momentum"] + else: + raise ValueError(f"Optimizer {optimizer_name} is not supported by Megatron-LM") + + self.megatron_lm_default_args["lr"] = optimizer.defaults["lr"] + self.megatron_lm_default_args["weight_decay"] = optimizer.defaults["weight_decay"] + + def set_scheduler_args(self, scheduler): + if self.train_iters is None: + self.train_iters = scheduler.total_num_steps // self.megatron_lm_default_args["data_parallel_size"] + if self.train_samples is not None: + self.train_samples = None + warnings.warn( + "Ignoring `train_samples` as `train_iters` based on scheduler is being used for training." + ) + if self.lr_warmup_iters is None: + self.lr_warmup_iters = scheduler.warmup_num_steps // self.megatron_lm_default_args["data_parallel_size"] + if self.lr_warmup_samples is not None: + warnings.warn( + "Ignoring `lr_warmup_samples` as `lr_warmup_iters` based on scheduler is being used for training." + ) + self.lr_warmup_samples = 0 + + self.megatron_lm_default_args["train_iters"] = self.train_iters + self.megatron_lm_default_args["lr_warmup_iters"] = self.lr_warmup_iters + self.megatron_lm_default_args["train_samples"] = self.train_samples + self.megatron_lm_default_args["lr_warmup_samples"] = self.lr_warmup_samples + self.megatron_lm_default_args["lr_decay_iters"] = self.lr_decay_iters + self.megatron_lm_default_args["lr_decay_samples"] = self.lr_decay_samples + self.megatron_lm_default_args["lr_warmup_fraction"] = self.lr_warmup_fraction + self.megatron_lm_default_args["lr_decay_style"] = self.lr_decay_style + self.megatron_lm_default_args["weight_decay_incr_style"] = self.weight_decay_incr_style + self.megatron_lm_default_args["start_weight_decay"] = self.start_weight_decay + self.megatron_lm_default_args["end_weight_decay"] = self.end_weight_decay + self.megatron_lm_default_args["min_lr"] = self.min_lr + + def set_tensorboard_logging_options(self): + from megatron.training.arguments import _add_logging_args + + parser = argparse.ArgumentParser() + parser = _add_logging_args(parser) + logging_args = parser.parse_known_args() + self.dataset_args = vars(logging_args[0]) + for key, value in self.dataset_args.items(): + if key.startswith("log_"): + self.megatron_lm_default_args[key] = True + elif key.startswith("no_log_"): + self.megatron_lm_default_args[key.replace("no_", "")] = True + + +MODEL_CONFIGS_TO_MEGATRON_PARSERS = {} + + +def add_model_config_to_megatron_parser(model_type: str): + def add_model_config_parser_helper(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + MODEL_CONFIGS_TO_MEGATRON_PARSERS[model_type] = func + return wrapper + + return add_model_config_parser_helper + + +@add_model_config_to_megatron_parser("megatron-bert") +def parse_bert_config(megatron_lm_plugin, model, batch_data): + model_type_name = "bert" + num_layers = model.config.num_hidden_layers + hidden_size = model.config.hidden_size + num_attention_heads = model.config.num_attention_heads + max_position_embeddings = model.config.max_position_embeddings + num_labels = model.config.num_labels + orig_vocab_size = model.config.vocab_size + pretraining_flag = False + if "maskedlm" in model.__class__.__name__.lower(): + pretraining_flag = True + if megatron_lm_plugin.seq_length is not None: + if megatron_lm_plugin.encoder_seq_length is not None: + warnings.warn("Both `seq_length` and `encoder_seq_length` are set. Using `encoder_seq_length`.") + megatron_lm_plugin.seq_length = megatron_lm_plugin.encoder_seq_length + elif megatron_lm_plugin.encoder_seq_length is not None: + megatron_lm_plugin.seq_length = megatron_lm_plugin.encoder_seq_length + elif batch_data is not None: + megatron_lm_plugin.seq_length = batch_data["input_ids"].shape[1] + else: + megatron_lm_plugin.seq_length = max_position_embeddings + megatron_lm_plugin.megatron_lm_default_args["seq_length"] = megatron_lm_plugin.seq_length + megatron_lm_plugin.megatron_lm_default_args["model_type_name"] = model_type_name + megatron_lm_plugin.megatron_lm_default_args["num_layers"] = num_layers + megatron_lm_plugin.megatron_lm_default_args["hidden_size"] = hidden_size + megatron_lm_plugin.megatron_lm_default_args["num_attention_heads"] = num_attention_heads + megatron_lm_plugin.megatron_lm_default_args["max_position_embeddings"] = max_position_embeddings + megatron_lm_plugin.megatron_lm_default_args["pretraining_flag"] = pretraining_flag + megatron_lm_plugin.megatron_lm_default_args["orig_vocab_size"] = orig_vocab_size + megatron_lm_plugin.megatron_lm_default_args["model_return_dict"] = model.config.return_dict + megatron_lm_plugin.megatron_lm_default_args["num_labels"] = num_labels + + +@add_model_config_to_megatron_parser("gpt2") +def parse_gpt2_config(megatron_lm_plugin, model, batch_data): + model_type_name = "gpt" + num_layers = model.config.n_layer + hidden_size = model.config.n_embd + num_attention_heads = model.config.n_head + max_position_embeddings = model.config.n_positions + orig_vocab_size = model.config.vocab_size + pretraining_flag = True + if megatron_lm_plugin.seq_length is not None: + if megatron_lm_plugin.decoder_seq_length is not None: + warnings.warn("Both `seq_length` and `decoder_seq_length` are set. Using `decoder_seq_length`.") + megatron_lm_plugin.seq_length = megatron_lm_plugin.decoder_seq_length + elif megatron_lm_plugin.decoder_seq_length is not None: + megatron_lm_plugin.seq_length = megatron_lm_plugin.decoder_seq_length + elif batch_data is not None: + megatron_lm_plugin.seq_length = batch_data["input_ids"].shape[1] + else: + megatron_lm_plugin.seq_length = max_position_embeddings + megatron_lm_plugin.megatron_lm_default_args["seq_length"] = megatron_lm_plugin.seq_length + megatron_lm_plugin.megatron_lm_default_args["return_logits"] = megatron_lm_plugin.return_logits + megatron_lm_plugin.megatron_lm_default_args["tokenizer_type"] = "GPT2BPETokenizer" + megatron_lm_plugin.megatron_lm_default_args["model_type_name"] = model_type_name + megatron_lm_plugin.megatron_lm_default_args["num_layers"] = num_layers + megatron_lm_plugin.megatron_lm_default_args["hidden_size"] = hidden_size + megatron_lm_plugin.megatron_lm_default_args["num_attention_heads"] = num_attention_heads + megatron_lm_plugin.megatron_lm_default_args["max_position_embeddings"] = max_position_embeddings + megatron_lm_plugin.megatron_lm_default_args["pretraining_flag"] = pretraining_flag + megatron_lm_plugin.megatron_lm_default_args["orig_vocab_size"] = orig_vocab_size + megatron_lm_plugin.megatron_lm_default_args["model_return_dict"] = model.config.return_dict + + +@add_model_config_to_megatron_parser("t5") +def parse_t5_config(megatron_lm_plugin, model, batch_data): + model_type_name = "t5" + num_layers = model.config.num_layers + hidden_size = model.config.d_model + num_attention_heads = model.config.num_heads + max_position_embeddings = model.config.n_positions if hasattr(model.config, "n_positions") else 1024 + orig_vocab_size = model.config.vocab_size + pretraining_flag = True + if megatron_lm_plugin.encoder_seq_length is None: + if batch_data is not None: + megatron_lm_plugin.encoder_seq_length = batch_data["input_ids"].shape[1] + else: + megatron_lm_plugin.encoder_seq_length = max_position_embeddings + if megatron_lm_plugin.decoder_seq_length is None: + if batch_data is not None: + megatron_lm_plugin.decoder_seq_length = batch_data["labels"].shape[1] + else: + megatron_lm_plugin.decoder_seq_length = max_position_embeddings + megatron_lm_plugin.megatron_lm_default_args["encoder_seq_length"] = megatron_lm_plugin.encoder_seq_length + megatron_lm_plugin.megatron_lm_default_args["decoder_seq_length"] = megatron_lm_plugin.decoder_seq_length + megatron_lm_plugin.megatron_lm_default_args["model_type_name"] = model_type_name + megatron_lm_plugin.megatron_lm_default_args["num_layers"] = num_layers + megatron_lm_plugin.megatron_lm_default_args["hidden_size"] = hidden_size + megatron_lm_plugin.megatron_lm_default_args["num_attention_heads"] = num_attention_heads + megatron_lm_plugin.megatron_lm_default_args["max_position_embeddings"] = max_position_embeddings + megatron_lm_plugin.megatron_lm_default_args["pretraining_flag"] = pretraining_flag + megatron_lm_plugin.megatron_lm_default_args["orig_vocab_size"] = orig_vocab_size + megatron_lm_plugin.megatron_lm_default_args["model_return_dict"] = model.config.return_dict + + +@add_model_config_to_megatron_parser("llama") +def parse_llama_config(megatron_lm_plugin, model, batch_data): + model_type_name = "gpt" + num_layers = model.config.num_hidden_layers + pretraining_flag = True + hidden_size = model.config.hidden_size + num_attention_heads = model.config.num_attention_heads + orig_vocab_size = model.config.vocab_size + + max_position_embeddings = model.config.max_position_embeddings + seq_length = getattr(model.config, "max_sequence_length", None) + if megatron_lm_plugin.seq_length is None: + if seq_length is not None: + megatron_lm_plugin.seq_length = seq_length + elif megatron_lm_plugin.decoder_seq_length is not None: + megatron_lm_plugin.seq_length = megatron_lm_plugin.decoder_seq_length + elif batch_data is not None: + megatron_lm_plugin.seq_length = batch_data["input_ids"].shape[1] + else: + megatron_lm_plugin.seq_length = max_position_embeddings + + megatron_lm_plugin.megatron_lm_default_args["return_logits"] = megatron_lm_plugin.return_logits + megatron_lm_plugin.megatron_lm_default_args["tokenizer_type"] = "Llama2Tokenizer" + megatron_lm_plugin.megatron_lm_default_args["model_type_name"] = model_type_name + megatron_lm_plugin.megatron_lm_default_args["num_layers"] = num_layers + megatron_lm_plugin.megatron_lm_default_args["pretraining_flag"] = pretraining_flag + megatron_lm_plugin.megatron_lm_default_args["hidden_size"] = hidden_size + megatron_lm_plugin.megatron_lm_default_args["num_attention_heads"] = num_attention_heads + megatron_lm_plugin.megatron_lm_default_args["orig_vocab_size"] = orig_vocab_size + megatron_lm_plugin.megatron_lm_default_args["max_position_embeddings"] = max_position_embeddings + megatron_lm_plugin.megatron_lm_default_args["seq_length"] = megatron_lm_plugin.seq_length + megatron_lm_plugin.megatron_lm_default_args["model_return_dict"] = model.config.return_dict + + +@add_model_config_to_megatron_parser("glm4_moe") +def parse_glm4_moe_config(megatron_lm_plugin, model, batch_data): + model_type_name = "gpt" + num_layers = model.config.num_hidden_layers + pretraining_flag = False + hidden_size = model.config.hidden_size + num_attention_heads = model.config.num_attention_heads + orig_vocab_size = model.config.vocab_size + + max_position_embeddings = model.config.max_position_embeddings + seq_length = getattr(model.config, "max_sequence_length", None) + if megatron_lm_plugin.seq_length is None: + if seq_length is not None: + megatron_lm_plugin.seq_length = seq_length + elif megatron_lm_plugin.decoder_seq_length is not None: + megatron_lm_plugin.seq_length = megatron_lm_plugin.decoder_seq_length + elif batch_data is not None: + megatron_lm_plugin.seq_length = batch_data["input_ids"].shape[1] + else: + megatron_lm_plugin.seq_length = max_position_embeddings + + megatron_lm_plugin.megatron_lm_default_args["return_logits"] = megatron_lm_plugin.return_logits + megatron_lm_plugin.megatron_lm_default_args["tokenizer_type"] = "HuggingFaceTokenizer" + megatron_lm_plugin.megatron_lm_default_args["model_type_name"] = model_type_name + megatron_lm_plugin.megatron_lm_default_args["num_layers"] = num_layers + megatron_lm_plugin.megatron_lm_default_args["pretraining_flag"] = pretraining_flag + megatron_lm_plugin.megatron_lm_default_args["hidden_size"] = hidden_size + megatron_lm_plugin.megatron_lm_default_args["num_attention_heads"] = num_attention_heads + megatron_lm_plugin.megatron_lm_default_args["kv_channels"] = model.config.head_dim + megatron_lm_plugin.megatron_lm_default_args["orig_vocab_size"] = orig_vocab_size + megatron_lm_plugin.megatron_lm_default_args["max_position_embeddings"] = max_position_embeddings + megatron_lm_plugin.megatron_lm_default_args["seq_length"] = megatron_lm_plugin.seq_length + megatron_lm_plugin.megatron_lm_default_args["model_return_dict"] = model.config.return_dict + megatron_lm_plugin.megatron_lm_default_args["position_embedding_type"] = "rope" + megatron_lm_plugin.megatron_lm_default_args["original_model_type"] = model.config.model_type + megatron_lm_plugin.megatron_lm_default_args["qk_layernorm"] = ( + model.config.use_qk_norm + ) # this is true for glm4.5 but False for glm4.5-air. + megatron_lm_plugin.megatron_lm_default_args["add_bias_linear"] = False + megatron_lm_plugin.megatron_lm_default_args["group_query_attention"] = True + megatron_lm_plugin.megatron_lm_default_args["num_query_groups"] = model.config.num_key_value_heads + megatron_lm_plugin.megatron_lm_default_args["ffn_hidden_size"] = model.config.intermediate_size + megatron_lm_plugin.megatron_lm_default_args["add_qkv_bias"] = True + megatron_lm_plugin.megatron_lm_default_args["normalization"] = "RMSNorm" + megatron_lm_plugin.megatron_lm_default_args["rotary-percent"] = 0.5 + megatron_lm_plugin.megatron_lm_default_args["swiglu"] = True + megatron_lm_plugin.megatron_lm_default_args["moe_ffn_hidden_size"] = model.config.moe_intermediate_size + megatron_lm_plugin.megatron_lm_default_args["moe_shared_expert_intermediate_size"] = ( + model.config.moe_intermediate_size + ) + megatron_lm_plugin.megatron_lm_default_args["moe_router_pre_softmax"] = True + megatron_lm_plugin.megatron_lm_default_args["moe_router_score_function"] = "sigmoid" + megatron_lm_plugin.megatron_lm_default_args["moe_router_enable_expert_bias"] = True + megatron_lm_plugin.megatron_lm_default_args["moe_router_bias_update_rate"] = 0 + megatron_lm_plugin.megatron_lm_default_args["moe_router_load_balancing_type"] = "seq_aux_loss" + megatron_lm_plugin.megatron_lm_default_args["moe_token_dispatcher_type"] = "alltoall" + megatron_lm_plugin.megatron_lm_default_args["moe_router_topk"] = model.config.num_experts_per_tok + megatron_lm_plugin.megatron_lm_default_args["moe_router_topk_scaling_factor"] = model.config.routed_scaling_factor + megatron_lm_plugin.megatron_lm_default_args["moe_layer_freq"] = [0] * model.config.first_k_dense_replace + [1] * ( + model.config.num_hidden_layers - model.config.first_k_dense_replace + ) + megatron_lm_plugin.megatron_lm_default_args["num_experts"] = model.config.n_routed_experts + megatron_lm_plugin.megatron_lm_default_args["moe_grouped_gemm"] = True + megatron_lm_plugin.megatron_lm_default_args["moe_router_dtype"] = "fp32" + megatron_lm_plugin.megatron_lm_default_args["moe_permute_fusion"] = True + megatron_lm_plugin.megatron_lm_default_args["moe_aux_loss_coeff"] = 0 + megatron_lm_plugin.megatron_lm_default_args["rotary_base"] = model.config.rope_theta + megatron_lm_plugin.megatron_lm_default_args["rope_type"] = "rope" + megatron_lm_plugin.megatron_lm_default_args["rotary_percent"] = model.config.partial_rotary_factor + megatron_lm_plugin.megatron_lm_default_args["norm_epsilon"] = 1e-3 + megatron_lm_plugin.megatron_lm_default_args["use_flash_attn"] = True + megatron_lm_plugin.megatron_lm_default_args["eos_token_id"] = model.config.eos_token_id + if getattr(model.config, "fp8_param", False): + megatron_lm_plugin.megatron_lm_default_args["fp8"] = model.config.fp8 + megatron_lm_plugin.megatron_lm_default_args["fp8_param"] = model.config.fp8_param + megatron_lm_plugin.megatron_lm_default_args["fp8_param_gather"] = model.config.fp8_param_gather + megatron_lm_plugin.megatron_lm_default_args["fp8_recipe"] = model.config.fp8_recipe + megatron_lm_plugin.megatron_lm_default_args["bf16"] = model.config.bf16 + megatron_lm_plugin.megatron_lm_default_args[ + "untie_embeddings_and_output_weights" + ] = not model.config.tie_word_embeddings + logger.info(f"Parsed GLM4 MoE config: {megatron_lm_plugin.megatron_lm_default_args}") + + +@dataclass +class BnbQuantizationConfig: + """ + A plugin to enable BitsAndBytes 4bit and 8bit quantization + + Args: + load_in_8bit (`bool`, defaults to `False`): + Enable 8bit quantization. + llm_int8_threshold (`float`, defaults to `6.0`): + Value of the outliner threshold. Only relevant when `load_in_8bit=True`. + load_in_4bit (`bool`, defaults to `False`): + Enable 4bit quantization. + bnb_4bit_quant_type (`str`, defaults to `fp4`): + Set the quantization data type in the `bnb.nn.Linear4Bit` layers. Options are {'fp4','np4'}. + bnb_4bit_use_double_quant (`bool`, defaults to `False`): + Enable nested quantization where the quantization constants from the first quantization are quantized + again. + bnb_4bit_compute_dtype (`bool`, defaults to `fp16`): + This sets the computational type which might be different than the input time. For example, inputs might be + fp32, but computation can be set to bf16 for speedups. Options are {'fp32','fp16','bf16'}. + torch_dtype (`torch.dtype`, defaults to `None`): + This sets the dtype of the remaining non quantized layers. `bitsandbytes` library suggests to set the value + to `torch.float16` for 8 bit model and use the same dtype as the compute dtype for 4 bit model. + skip_modules (`List[str]`, defaults to `None`): + An explicit list of the modules that we don't quantize. The dtype of these modules will be `torch_dtype`. + keep_in_fp32_modules (`List`, defaults to `None`): + An explicit list of the modules that we don't quantize. We keep them in `torch.float32`. + """ + + load_in_8bit: bool = field(default=False, metadata={"help": "enable 8bit quantization."}) + + llm_int8_threshold: float = field( + default=6.0, + metadata={"help": "value of the outliner threshold. only relevant when load_in_8bit=True"}, + ) + + load_in_4bit: bool = field(default=False, metadata={"help": "enable 4bit quantization."}) + + bnb_4bit_quant_type: str = field( + default="fp4", + metadata={ + "help": "set the quantization data type in the `bnb.nn.Linear4Bit` layers. Options are {'fp4','nf4'}." + }, + ) + + bnb_4bit_use_double_quant: bool = field( + default=False, + metadata={ + "help": "enable nested quantization where the quantization constants from the first quantization are quantized again." + }, + ) + + bnb_4bit_compute_dtype: str = field( + default="fp16", + metadata={ + "help": "This sets the computational type which might be different than the input time. For example, inputs might be " + "fp32, but computation can be set to bf16 for speedups. Options are {'fp32','fp16','bf16'}." + }, + ) + + torch_dtype: torch.dtype = field( + default=None, + metadata={ + "help": "this sets the dtype of the remaining non quantized layers. `bitsandbytes` library suggests to set the value" + "to `torch.float16` for 8 bit model and use the same dtype as the compute dtype for 4 bit model " + }, + ) + + skip_modules: list[str] = field( + default=None, + metadata={ + "help": "an explicit list of the modules that we don't quantize. The dtype of these modules will be `torch_dtype`." + }, + ) + + keep_in_fp32_modules: list[str] = field( + default=None, + metadata={"help": "an explicit list of the modules that we don't quantize. We keep them in `torch.float32`."}, + ) + + def __post_init__(self): + """ + Safety checker that arguments are correct - also replaces some NoneType arguments with their default values. + """ + if not isinstance(self.load_in_8bit, bool): + raise ValueError("load_in_8bit must be a boolean") + + if not isinstance(self.load_in_4bit, bool): + raise ValueError("load_in_4bit must be a boolean") + + if self.load_in_4bit and self.load_in_8bit: + raise ValueError("load_in_4bit and load_in_8bit can't be both True") + + if not self.load_in_4bit and not self.load_in_8bit: + raise ValueError("load_in_4bit and load_in_8bit can't be both False") + + if not isinstance(self.llm_int8_threshold, (int, float)): + raise ValueError("llm_int8_threshold must be a float or an int") + + if not isinstance(self.bnb_4bit_quant_type, str): + raise ValueError("bnb_4bit_quant_type must be a string") + elif self.bnb_4bit_quant_type not in ["fp4", "nf4"]: + raise ValueError(f"bnb_4bit_quant_type must be in ['fp4','nf4'] but found {self.bnb_4bit_quant_type}") + + if not isinstance(self.bnb_4bit_use_double_quant, bool): + raise ValueError("bnb_4bit_use_double_quant must be a boolean") + + if isinstance(self.bnb_4bit_compute_dtype, str): + if self.bnb_4bit_compute_dtype == "fp32": + self.bnb_4bit_compute_dtype = torch.float32 + elif self.bnb_4bit_compute_dtype == "fp16": + self.bnb_4bit_compute_dtype = torch.float16 + elif self.bnb_4bit_compute_dtype == "bf16": + self.bnb_4bit_compute_dtype = torch.bfloat16 + else: + raise ValueError( + f"bnb_4bit_compute_dtype must be in ['fp32','fp16','bf16'] but found {self.bnb_4bit_compute_dtype}" + ) + elif not isinstance(self.bnb_4bit_compute_dtype, torch.dtype): + raise ValueError("bnb_4bit_compute_dtype must be a string or a torch.dtype") + + if self.skip_modules is not None and not isinstance(self.skip_modules, list): + raise ValueError("skip_modules must be a list of strings") + + if self.keep_in_fp32_modules is not None and not isinstance(self.keep_in_fp32_modules, list): + raise ValueError("keep_in_fp_32_modules must be a list of strings") + + if self.load_in_4bit: + self.target_dtype = CustomDtype.INT4 + + if self.load_in_8bit: + self.target_dtype = torch.int8 + + if self.load_in_4bit and self.llm_int8_threshold != 6.0: + warnings.warn("llm_int8_threshold can only be used for model loaded in 8bit") + + if isinstance(self.torch_dtype, str): + if self.torch_dtype == "fp32": + self.torch_dtype = torch.float32 + elif self.torch_dtype == "fp16": + self.torch_dtype = torch.float16 + elif self.torch_dtype == "bf16": + self.torch_dtype = torch.bfloat16 + else: + raise ValueError(f"torch_dtype must be in ['fp32','fp16','bf16'] but found {self.torch_dtype}") + if self.load_in_8bit and self.torch_dtype is None: + self.torch_dtype = torch.float16 + + if self.load_in_4bit and self.torch_dtype is None: + self.torch_dtype = self.bnb_4bit_compute_dtype + + if not isinstance(self.torch_dtype, torch.dtype): + raise ValueError("torch_dtype must be a torch.dtype") + + +def get_module_class_from_name(module, name): + """ + Gets a class from a module by its name. + + Args: + module (`torch.nn.Module`): The module to get the class from. + name (`str`): The name of the class. + """ + modules_children = list(module.children()) + if module.__class__.__name__ == name: + return module.__class__ + elif len(modules_children) == 0: + return + else: + for child_module in modules_children: + module_class = get_module_class_from_name(child_module, name) + if module_class is not None: + return module_class diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/deepspeed.py b/kbidu_env/Lib/site-packages/accelerate/utils/deepspeed.py new file mode 100644 index 0000000000000000000000000000000000000000..22db891c63d9bd48691acd87a15a206c270017a9 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/deepspeed.py @@ -0,0 +1,385 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import os +from copy import deepcopy + +from torch import optim + +from ..optimizer import AcceleratedOptimizer +from ..scheduler import AcceleratedScheduler +from .dataclasses import DistributedType +from .imports import is_bnb_available +from .versions import compare_versions + + +def map_pytorch_optim_to_deepspeed(optimizer): + """ + Args: + optimizer: torch.optim.Optimizer + + Returns the DeepSeedCPUOptimizer (deepspeed.ops) version of the optimizer. + """ + + defaults = {k: v for k, v in optimizer.defaults.items() if k in ["lr", "weight_decay"]} + + # Select the DeepSpeedCPUOptimizer based on the original optimizer class. + # DeepSpeedCPUAdam is the default + from deepspeed.ops.adam import DeepSpeedCPUAdam + + optimizer_class = DeepSpeedCPUAdam + + # For DeepSpeedCPUAdam (adamw_mode) + if compare_versions("deepspeed", ">=", "0.3.1"): + defaults["adamw_mode"] = False + is_adaw = isinstance(optimizer, optim.AdamW) + + if is_bnb_available() and not is_adaw: + import bitsandbytes.optim as bnb_opt + + if isinstance(optimizer, (bnb_opt.AdamW, bnb_opt.AdamW32bit)): + try: + is_adaw = optimizer.optim_bits == 32 + except AttributeError: + is_adaw = optimizer.args.optim_bits == 32 + else: + is_adaw = False + + if is_adaw: + defaults["adamw_mode"] = True + + # For DeepSpeedCPUAdagrad + if compare_versions("deepspeed", ">=", "0.5.5"): + # Check if the optimizer is PyTorch's Adagrad. + is_ada = isinstance(optimizer, optim.Adagrad) + # If not, and bitsandbytes is available, + # # check if the optimizer is the 32-bit bitsandbytes Adagrad. + if is_bnb_available() and not is_ada: + import bitsandbytes.optim as bnb_opt + + if isinstance(optimizer, (bnb_opt.Adagrad, bnb_opt.Adagrad32bit)): + try: + is_ada = optimizer.optim_bits == 32 + except AttributeError: + is_ada = optimizer.args.optim_bits == 32 + if is_ada: + from deepspeed.ops.adagrad import DeepSpeedCPUAdagrad + + optimizer_class = DeepSpeedCPUAdagrad + + # For DeepSpeedCPULion + if is_bnb_available(min_version="0.38.0") and compare_versions("deepspeed", ">=", "0.11.0"): + from bitsandbytes.optim import Lion, Lion32bit + + if isinstance(optimizer, (Lion, Lion32bit)): + try: + is_bnb_32bits = optimizer.optim_bits == 32 + except AttributeError: + is_bnb_32bits = optimizer.args.optim_bits == 32 + if is_bnb_32bits: + from deepspeed.ops.lion import DeepSpeedCPULion + + optimizer_class = DeepSpeedCPULion + + return optimizer_class(optimizer.param_groups, **defaults) + + +def get_active_deepspeed_plugin(state): + """ + Returns the currently active DeepSpeedPlugin. + + Raises: + ValueError: If DeepSpeed was not enabled and this function is called. + """ + if state.distributed_type != DistributedType.DEEPSPEED: + raise ValueError( + "Couldn't retrieve the active `DeepSpeedPlugin` as none were enabled. " + "Please make sure that either `Accelerator` is configured for `deepspeed` " + "or make sure that the desired `DeepSpeedPlugin` has been enabled (`AcceleratorState().select_deepspeed_plugin(name)`) " + "before calling this function." + ) + if not isinstance(state.deepspeed_plugins, dict): + return state.deepspeed_plugins + return next(plugin for plugin in state.deepspeed_plugins.values() if plugin.selected) + + +class HfDeepSpeedConfig: + """ + This object contains a DeepSpeed configuration dictionary and can be quickly queried for things like zero stage. + + A `weakref` of this object is stored in the module's globals to be able to access the config from areas where + things like the Trainer object is not available (e.g. `from_pretrained` and `_get_resized_embeddings`). Therefore + it's important that this object remains alive while the program is still running. + + [`Trainer`] uses the `HfTrainerDeepSpeedConfig` subclass instead. That subclass has logic to sync the configuration + with values of [`TrainingArguments`] by replacing special placeholder values: `"auto"`. Without this special logic + the DeepSpeed configuration is not modified in any way. + + Args: + config_file_or_dict (`Union[str, Dict]`): path to DeepSpeed config file or dict. + + """ + + def __init__(self, config_file_or_dict): + if isinstance(config_file_or_dict, dict): + # Don't modify user's data should they want to reuse it (e.g. in tests), because once we + # modified it, it will not be accepted here again, since `auto` values would have been overridden + config = deepcopy(config_file_or_dict) + elif os.path.exists(config_file_or_dict): + with open(config_file_or_dict, encoding="utf-8") as f: + config = json.load(f) + else: + try: + try: + # First try parsing as JSON directly + config = json.loads(config_file_or_dict) + except json.JSONDecodeError: + # If that fails, try base64 decoding + config_decoded = base64.urlsafe_b64decode(config_file_or_dict).decode("utf-8") + config = json.loads(config_decoded) + except (UnicodeDecodeError, AttributeError, ValueError): + raise ValueError( + f"Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}" + ) + + self.config = config + + self.set_stage_and_offload() + + def set_stage_and_offload(self): + # zero stage - this is done as early as possible, before model is created, to allow + # ``is_deepspeed_zero3_enabled`` query and getting to the early deepspeed config object + # during ``zero.Init()`` which needs to know the dtype, and some other hparams. + self._stage = self.get_value("zero_optimization.stage", -1) + + # offload + self._offload = False + if self.is_zero2() or self.is_zero3(): + offload_devices_valid = set(["cpu", "nvme"]) + offload_devices = set( + [ + self.get_value("zero_optimization.offload_optimizer.device"), + self.get_value("zero_optimization.offload_param.device"), + ] + ) + if len(offload_devices & offload_devices_valid) > 0: + self._offload = True + + def find_config_node(self, ds_key_long): + config = self.config + + # find the config node of interest if it exists + nodes = ds_key_long.split(".") + ds_key = nodes.pop() + for node in nodes: + config = config.get(node) + if config is None: + return None, ds_key + + return config, ds_key + + def get_value(self, ds_key_long, default=None): + """ + Returns the set value or `default` if no value is set + """ + config, ds_key = self.find_config_node(ds_key_long) + if config is None: + return default + return config.get(ds_key, default) + + def del_config_sub_tree(self, ds_key_long, must_exist=False): + """ + Deletes a sub-section of the config file if it's found. + + Unless `must_exist` is `True` the section doesn't have to exist. + """ + config = self.config + + # find the config node of interest if it exists + nodes = ds_key_long.split(".") + for node in nodes: + parent_config = config + config = config.get(node) + if config is None: + if must_exist: + raise ValueError(f"Can't find {ds_key_long} entry in the config: {self.config}") + else: + return + + # if found remove it + if parent_config is not None: + parent_config.pop(node) + + def is_true(self, ds_key_long): + """ + Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very + specific question of whether the value is set to `True` (and it's not set to `False`` or isn't set). + + """ + value = self.get_value(ds_key_long) + return False if value is None else bool(value) + + def is_false(self, ds_key_long): + """ + Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very + specific question of whether the value is set to `False` (and it's not set to `True`` or isn't set). + """ + value = self.get_value(ds_key_long) + return False if value is None else not bool(value) + + def is_zero2(self): + return self._stage == 2 + + def is_zero3(self): + return self._stage == 3 + + def is_offload(self): + return self._offload + + +class DeepSpeedEngineWrapper: + """ + Internal wrapper for deepspeed.runtime.engine.DeepSpeedEngine. This is used to follow conventional training loop. + + Args: + engine (deepspeed.runtime.engine.DeepSpeedEngine): deepspeed engine to wrap + """ + + def __init__(self, engine): + self.engine = engine + + def backward(self, loss, sync_gradients=True, **kwargs): + # Set gradient accumulation boundary based on Accelerate's sync_gradients state + # This tells DeepSpeed whether this is the final micro-batch before gradient sync + self.engine.set_gradient_accumulation_boundary(is_boundary=sync_gradients) + + # runs backpropagation and handles mixed precision + self.engine.backward(loss, **kwargs) + + # Only perform step and related operations at gradient accumulation boundaries + if sync_gradients: + # Deepspeed's `engine.step` performs the following operations: + # - gradient accumulation check + # - gradient clipping + # - optimizer step + # - zero grad + # - checking overflow + # - lr_scheduler step (only if engine.lr_scheduler is not None) + self.engine.step() + # and this plugin overrides the above calls with no-ops when Accelerate runs under + # Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple + # training loop that works transparently under many training regimes. + + def get_global_grad_norm(self): + """Get the global gradient norm from DeepSpeed engine.""" + grad_norm = self.engine.get_global_grad_norm() + # Convert to scalar if it's a tensor + if hasattr(grad_norm, "item"): + return grad_norm.item() + return grad_norm + + +class DeepSpeedOptimizerWrapper(AcceleratedOptimizer): + """ + Internal wrapper around a deepspeed optimizer. + + Args: + optimizer (`torch.optim.optimizer.Optimizer`): + The optimizer to wrap. + """ + + def __init__(self, optimizer): + super().__init__(optimizer, device_placement=False, scaler=None) + self.__has_overflow__ = hasattr(self.optimizer, "overflow") + + def zero_grad(self, set_to_none=None): + pass # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed + + def step(self): + pass # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed + + @property + def step_was_skipped(self): + """Whether or not the optimizer step was done, or skipped because of gradient overflow.""" + if self.__has_overflow__: + return self.optimizer.overflow + return False + + +class DeepSpeedSchedulerWrapper(AcceleratedScheduler): + """ + Internal wrapper around a deepspeed scheduler. + + Args: + scheduler (`torch.optim.lr_scheduler.LambdaLR`): + The scheduler to wrap. + optimizers (one or a list of `torch.optim.Optimizer`): + """ + + def __init__(self, scheduler, optimizers): + super().__init__(scheduler, optimizers) + + def step(self): + pass # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed + + +class DummyOptim: + """ + Dummy optimizer presents model parameters or param groups, this is primarily used to follow conventional training + loop when optimizer config is specified in the deepspeed config file. + + Args: + lr (float): + Learning rate. + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + weight_decay (float): + Weight decay. + **kwargs (additional keyword arguments, *optional*): + Other arguments. + """ + + def __init__(self, params, lr=0.001, weight_decay=0, **kwargs): + self.params = params + self.lr = lr + self.weight_decay = weight_decay + self.kwargs = kwargs + + +class DummyScheduler: + """ + Dummy scheduler presents model parameters or param groups, this is primarily used to follow conventional training + loop when scheduler config is specified in the deepspeed config file. + + Args: + optimizer (`torch.optim.optimizer.Optimizer`): + The optimizer to wrap. + total_num_steps (int, *optional*): + Total number of steps. + warmup_num_steps (int, *optional*): + Number of steps for warmup. + lr_scheduler_callable (callable, *optional*): + A callable function that creates an LR Scheduler. It accepts only one argument `optimizer`. + **kwargs (additional keyword arguments, *optional*): + Other arguments. + """ + + def __init__(self, optimizer, total_num_steps=None, warmup_num_steps=0, lr_scheduler_callable=None, **kwargs): + self.optimizer = optimizer + self.total_num_steps = total_num_steps + self.warmup_num_steps = warmup_num_steps + self.lr_scheduler_callable = lr_scheduler_callable + self.kwargs = kwargs diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/environment.py b/kbidu_env/Lib/site-packages/accelerate/utils/environment.py new file mode 100644 index 0000000000000000000000000000000000000000..6263ffa68babf0a04d974102b838b0482964a471 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/environment.py @@ -0,0 +1,512 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import math +import os +import platform +import subprocess +import sys +from contextlib import contextmanager +from dataclasses import dataclass, field +from functools import lru_cache, wraps +from shutil import which +from typing import Optional, Union + +import torch +from packaging.version import parse + + +logger = logging.getLogger(__name__) + + +def convert_dict_to_env_variables(current_env: dict): + """ + Verifies that all keys and values in `current_env` do not contain illegal keys or values, and returns a list of + strings as the result. + + Example: + ```python + >>> from accelerate.utils.environment import verify_env + + >>> env = {"ACCELERATE_DEBUG_MODE": "1", "BAD_ENV_NAME": ">> valid_env_items = verify_env(env) + >>> print(valid_env_items) + ["ACCELERATE_DEBUG_MODE=1\n", "OTHER_ENV=2\n"] + ``` + """ + forbidden_chars = [";", "\n", "<", ">", " "] + valid_env_items = [] + for key, value in current_env.items(): + if all(char not in (key + value) for char in forbidden_chars) and len(key) >= 1 and len(value) >= 1: + valid_env_items.append(f"{key}={value}\n") + else: + logger.warning(f"WARNING: Skipping {key}={value} as it contains forbidden characters or missing values.") + return valid_env_items + + +def str_to_bool(value, to_bool: bool = False) -> Union[int, bool]: + """ + Converts a string representation of truth to `True` (1) or `False` (0). + + True values are `y`, `yes`, `t`, `true`, `on`, and `1`; False value are `n`, `no`, `f`, `false`, `off`, and `0`; + """ + value = value.lower() + if value in ("y", "yes", "t", "true", "on", "1"): + return 1 if not to_bool else True + elif value in ("n", "no", "f", "false", "off", "0"): + return 0 if not to_bool else False + else: + raise ValueError(f"invalid truth value {value}") + + +def get_int_from_env(env_keys, default): + """Returns the first positive env value found in the `env_keys` list or the default.""" + for e in env_keys: + val = int(os.environ.get(e, -1)) + if val >= 0: + return val + return default + + +def parse_flag_from_env(key, default=False): + """Returns truthy value for `key` from the env if available else the default.""" + value = os.environ.get(key, str(default)) + return str_to_bool(value) == 1 # As its name indicates `str_to_bool` actually returns an int... + + +def parse_choice_from_env(key, default="no"): + value = os.environ.get(key, str(default)) + return value + + +def are_libraries_initialized(*library_names: str) -> list[str]: + """ + Checks if any of `library_names` are imported in the environment. Will return any names that are. + """ + return [lib_name for lib_name in library_names if lib_name in sys.modules.keys()] + + +def get_current_device_type() -> tuple[str, str]: + """ + Determines the current device type and distributed type without initializing any device. + + This is particularly important when using fork-based multiprocessing, as device initialization + before forking can cause errors. + + The device detection order follows the same priority as state.py:_prepare_backend(): + MLU -> SDAA -> MUSA -> NPU -> HPU -> CUDA -> XPU + + Returns: + tuple[str, str]: A tuple of (device_type, distributed_type) + - device_type: The device string (e.g., "cuda", "npu", "xpu") + - distributed_type: The distributed type string (e.g., "MULTI_GPU", "MULTI_NPU") + + Example: + ```python + >>> device_type, distributed_type = get_current_device_type() + >>> print(device_type) # "cuda" + >>> print(distributed_type) # "MULTI_GPU" + ``` + """ + from .imports import ( + is_hpu_available, + is_mlu_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_xpu_available, + ) + + if is_mlu_available(): + return "mlu", "MULTI_MLU" + elif is_sdaa_available(): + return "sdaa", "MULTI_SDAA" + elif is_musa_available(): + return "musa", "MULTI_MUSA" + elif is_npu_available(): + return "npu", "MULTI_NPU" + elif is_hpu_available(): + return "hpu", "MULTI_HPU" + elif is_xpu_available(): + return "xpu", "MULTI_XPU" + elif is_neuron_available(): + return "neuron", "MULTI_NEURON" + else: + return "cuda", "MULTI_GPU" + + +def _nvidia_smi(): + """ + Returns the right nvidia-smi command based on the system. + """ + if platform.system() == "Windows": + # If platform is Windows and nvidia-smi can't be found in path + # try from systemd drive with default installation path + command = which("nvidia-smi") + if command is None: + command = f"{os.environ['systemdrive']}\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" + else: + command = "nvidia-smi" + return command + + +def get_gpu_info(): + """ + Gets GPU count and names using `nvidia-smi` instead of torch to not initialize CUDA. + + Largely based on the `gputil` library. + """ + # Returns as list of `n` GPUs and their names + output = subprocess.check_output( + [_nvidia_smi(), "--query-gpu=count,name", "--format=csv,noheader"], universal_newlines=True + ) + output = output.strip() + gpus = output.split(os.linesep) + # Get names from output + gpu_count = len(gpus) + gpu_names = [gpu.split(",")[1].strip() for gpu in gpus] + return gpu_names, gpu_count + + +def get_driver_version(): + """ + Returns the driver version + + In the case of multiple GPUs, will return the first. + """ + output = subprocess.check_output( + [_nvidia_smi(), "--query-gpu=driver_version", "--format=csv,noheader"], universal_newlines=True + ) + output = output.strip() + return output.split(os.linesep)[0] + + +def check_cuda_p2p_ib_support(): + """ + Checks if the devices being used have issues with P2P and IB communications, namely any consumer GPU hardware after + the 3090. + + Notably uses `nvidia-smi` instead of torch to not initialize CUDA. + """ + try: + device_names, device_count = get_gpu_info() + # As new consumer GPUs get released, add them to `unsupported_devices`` + unsupported_devices = {"RTX 40"} + if device_count > 1: + if any( + unsupported_device in device_name + for device_name in device_names + for unsupported_device in unsupported_devices + ): + # Check if they have the right driver version + acceptable_driver_version = "550.40.07" + current_driver_version = get_driver_version() + if parse(current_driver_version) < parse(acceptable_driver_version): + return False + return True + except Exception: + pass + return True + + +@lru_cache +def check_cuda_fp8_capability(): + """ + Checks if the current GPU available supports FP8. + + Notably might initialize `torch.cuda` to check. + """ + + try: + # try to get the compute capability from nvidia-smi + output = subprocess.check_output( + [_nvidia_smi(), "--query-gpu=compute_capability", "--format=csv,noheader"], universal_newlines=True + ) + output = output.strip() + # we take the first GPU's compute capability + compute_capability = tuple(map(int, output.split(os.linesep)[0].split("."))) + except Exception: + compute_capability = torch.cuda.get_device_capability() + + return compute_capability >= (8, 9) + + +@dataclass +class CPUInformation: + """ + Stores information about the CPU in a distributed environment. It contains the following attributes: + - rank: The rank of the current process. + - world_size: The total number of processes in the world. + - local_rank: The rank of the current process on the local node. + - local_world_size: The total number of processes on the local node. + """ + + rank: int = field(default=0, metadata={"help": "The rank of the current process."}) + world_size: int = field(default=1, metadata={"help": "The total number of processes in the world."}) + local_rank: int = field(default=0, metadata={"help": "The rank of the current process on the local node."}) + local_world_size: int = field(default=1, metadata={"help": "The total number of processes on the local node."}) + + +def get_cpu_distributed_information() -> CPUInformation: + """ + Returns various information about the environment in relation to CPU distributed training as a `CPUInformation` + dataclass. + """ + information = {} + information["rank"] = get_int_from_env(["RANK", "PMI_RANK", "OMPI_COMM_WORLD_RANK", "MV2_COMM_WORLD_RANK"], 0) + information["world_size"] = get_int_from_env( + ["WORLD_SIZE", "PMI_SIZE", "OMPI_COMM_WORLD_SIZE", "MV2_COMM_WORLD_SIZE"], 1 + ) + information["local_rank"] = get_int_from_env( + ["LOCAL_RANK", "MPI_LOCALRANKID", "OMPI_COMM_WORLD_LOCAL_RANK", "MV2_COMM_WORLD_LOCAL_RANK"], 0 + ) + information["local_world_size"] = get_int_from_env( + ["LOCAL_WORLD_SIZE", "MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], + 1, + ) + return CPUInformation(**information) + + +def _parse_cpu_list(cpu_list_str: str) -> list[int]: + """Parse a Linux sysfs-style CPU list (e.g. "0-7,16-23") into a list of ints.""" + cpus = [] + for part in cpu_list_str.split(","): + part = part.strip() + if not part: + continue + if "-" in part: + start, end = part.split("-") + cpus.extend(range(int(start), int(end) + 1)) + else: + cpus.append(int(part)) + return cpus + + +def override_numa_affinity(local_process_index: int, verbose: Optional[bool] = None) -> None: + """ + Overrides whatever NUMA affinity is set for the current process. This is very taxing and requires recalculating the + affinity to set, ideally you should use `utils.environment.set_numa_affinity` instead. + + Args: + local_process_index (int): + The index of the current process on the current server. + verbose (bool, *optional*): + Whether to log out the assignment of each CPU. If `ACCELERATE_DEBUG_MODE` is enabled, will default to True. + """ + if verbose is None: + verbose = parse_flag_from_env("ACCELERATE_DEBUG_MODE", False) + if torch.cuda.is_available(): + from accelerate.utils import is_amdsmi_available, is_pynvml_available, is_rocm_available + + affinity_to_set = None + + if is_rocm_available(): + if not is_amdsmi_available(): + raise ImportError( + "To set CPU affinity on ROCm GPUs the `amdsmi` package must be available. " + "It ships with ROCm; ensure the ROCm Python bindings are on PYTHONPATH." + ) + import amdsmi + + amdsmi.amdsmi_init() + try: + handles = amdsmi.amdsmi_get_processor_handles() + handle = handles[local_process_index] + numa_node = amdsmi.amdsmi_topo_get_numa_node_number(handle) + if numa_node is None or numa_node < 0: + # GPU is not bound to a NUMA node; fall back to all online CPUs + affinity_to_set = list(os.sched_getaffinity(0)) + else: + with open(f"/sys/devices/system/node/node{numa_node}/cpulist") as f: + cpu_list_str = f.read().strip() + affinity_to_set = _parse_cpu_list(cpu_list_str) + finally: + amdsmi.amdsmi_shut_down() + else: + if not is_pynvml_available(): + raise ImportError( + "To set CPU affinity on CUDA GPUs the `nvidia-ml-py` package must be available. (`pip install nvidia-ml-py`)" + ) + import pynvml as nvml + + # The below code is based on https://github.com/NVIDIA/DeepLearningExamples/blob/master/TensorFlow2/LanguageModeling/BERT/gpu_affinity.py + nvml.nvmlInit() + num_elements = math.ceil(os.cpu_count() / 64) + handle = nvml.nvmlDeviceGetHandleByIndex(local_process_index) + affinity_string = "" + for j in nvml.nvmlDeviceGetCpuAffinity(handle, num_elements): + # assume nvml returns list of 64 bit ints + affinity_string = f"{j:064b}{affinity_string}" + affinity_list = [int(x) for x in affinity_string] + affinity_list.reverse() # so core 0 is the 0th element + affinity_to_set = [i for i, e in enumerate(affinity_list) if e != 0] + + os.sched_setaffinity(0, affinity_to_set) + if verbose: + cpu_cores = os.sched_getaffinity(0) + logger.info(f"Assigning {len(cpu_cores)} cpu cores to process {local_process_index}: {cpu_cores}") + + +@lru_cache +def set_numa_affinity(local_process_index: int, verbose: Optional[bool] = None) -> None: + """ + Assigns the current process to a specific NUMA node. Ideally most efficient when having at least 2 cpus per node. + + This result is cached between calls. If you want to override it, please use + `accelerate.utils.environment.override_numa_afifnity`. + + Args: + local_process_index (int): + The index of the current process on the current server. + verbose (bool, *optional*): + Whether to print the new cpu cores assignment for each process. If `ACCELERATE_DEBUG_MODE` is enabled, will + default to True. + """ + override_numa_affinity(local_process_index=local_process_index, verbose=verbose) + + +@contextmanager +def clear_environment(): + """ + A context manager that will temporarily clear environment variables. + + When this context exits, the previous environment variables will be back. + + Example: + + ```python + >>> import os + >>> from accelerate.utils import clear_environment + + >>> os.environ["FOO"] = "bar" + >>> with clear_environment(): + ... print(os.environ) + ... os.environ["FOO"] = "new_bar" + ... print(os.environ["FOO"]) + {} + new_bar + + >>> print(os.environ["FOO"]) + bar + ``` + """ + _old_os_environ = os.environ.copy() + os.environ.clear() + + try: + yield + finally: + os.environ.clear() # clear any added keys, + os.environ.update(_old_os_environ) # then restore previous environment + + +@contextmanager +def patch_environment(**kwargs): + """ + A context manager that will add each keyword argument passed to `os.environ` and remove them when exiting. + + Will convert the values in `kwargs` to strings and upper-case all the keys. + + Example: + + ```python + >>> import os + >>> from accelerate.utils import patch_environment + + >>> with patch_environment(FOO="bar"): + ... print(os.environ["FOO"]) # prints "bar" + >>> print(os.environ["FOO"]) # raises KeyError + ``` + """ + existing_vars = {} + for key, value in kwargs.items(): + key = key.upper() + if key in os.environ: + existing_vars[key] = os.environ[key] + os.environ[key] = str(value) + + try: + yield + finally: + for key in kwargs: + key = key.upper() + if key in existing_vars: + # restore previous value + os.environ[key] = existing_vars[key] + else: + os.environ.pop(key, None) + + +def purge_accelerate_environment(func_or_cls): + """Decorator to clean up accelerate environment variables set by the decorated class or function. + + In some circumstances, calling certain classes or functions can result in accelerate env vars being set and not + being cleaned up afterwards. As an example, when calling: + + TrainingArguments(fp16=True, ...) + + The following env var will be set: + + ACCELERATE_MIXED_PRECISION=fp16 + + This can affect subsequent code, since the env var takes precedence over TrainingArguments(fp16=False). This is + especially relevant for unit testing, where we want to avoid the individual tests to have side effects on one + another. Decorate the unit test function or whole class with this decorator to ensure that after each test, the env + vars are cleaned up. This works for both unittest.TestCase and normal classes (pytest); it also works when + decorating the parent class. + + """ + prefix = "ACCELERATE_" + + @contextmanager + def env_var_context(): + # Store existing accelerate env vars + existing_vars = {k: v for k, v in os.environ.items() if k.startswith(prefix)} + try: + yield + finally: + # Restore original env vars or remove new ones + for key in [k for k in os.environ if k.startswith(prefix)]: + if key in existing_vars: + os.environ[key] = existing_vars[key] + else: + os.environ.pop(key, None) + + def wrap_function(func): + @wraps(func) + def wrapper(*args, **kwargs): + with env_var_context(): + return func(*args, **kwargs) + + wrapper._accelerate_is_purged_environment_wrapped = True + return wrapper + + if not isinstance(func_or_cls, type): + return wrap_function(func_or_cls) + + # Handle classes by wrapping test methods + def wrap_test_methods(test_class_instance): + for name in dir(test_class_instance): + if name.startswith("test"): + method = getattr(test_class_instance, name) + if callable(method) and not hasattr(method, "_accelerate_is_purged_environment_wrapped"): + setattr(test_class_instance, name, wrap_function(method)) + return test_class_instance + + # Handle inheritance + wrap_test_methods(func_or_cls) + func_or_cls.__init_subclass__ = classmethod(lambda cls, **kw: wrap_test_methods(cls)) + return func_or_cls diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/fsdp_utils.py b/kbidu_env/Lib/site-packages/accelerate/utils/fsdp_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5886eb9ef00f9c2c3a26089ebc25539105ae935c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/fsdp_utils.py @@ -0,0 +1,916 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import functools +import os +import re +import shutil +import warnings +from collections import defaultdict +from collections.abc import Iterable +from contextlib import nullcontext +from pathlib import Path +from typing import Callable, Union + +import torch + +from ..logging import get_logger +from .constants import FSDP_MODEL_NAME, OPTIMIZER_NAME, SAFE_WEIGHTS_NAME, WEIGHTS_NAME +from .dataclasses import get_module_class_from_name +from .modeling import get_non_persistent_buffers, is_peft_model +from .other import get_module_children_bottom_up, is_compiled_module, save +from .versions import is_torch_version + + +logger = get_logger(__name__) + + +def enable_fsdp_ram_efficient_loading(): + """ + Enables RAM efficient loading of Hugging Face models for FSDP in the environment. + """ + # Sets values for `transformers.modeling_utils.is_fsdp_enabled` + if "ACCELERATE_USE_FSDP" not in os.environ: + os.environ["ACCELERATE_USE_FSDP"] = "True" + os.environ["FSDP_CPU_RAM_EFFICIENT_LOADING"] = "True" + + +def disable_fsdp_ram_efficient_loading(): + """ + Disables RAM efficient loading of Hugging Face models for FSDP in the environment. + """ + os.environ["FSDP_CPU_RAM_EFFICIENT_LOADING"] = "False" + + +def _get_model_state_dict(model, adapter_only=False, sd_options=None): + if adapter_only and is_peft_model(model): + from peft import get_peft_model_state_dict + + return get_peft_model_state_dict(model, adapter_name=model.active_adapter) + + # Invariant: `sd_options` is not None only for FSDP2 + if sd_options is not None: + from torch.distributed.checkpoint.state_dict import get_model_state_dict + + return get_model_state_dict(model, options=sd_options) + else: + return model.state_dict() + + +def _set_model_state_dict(model, state_dict, adapter_only=False, sd_options=None): + if adapter_only and is_peft_model(model): + from peft import set_peft_model_state_dict + + return set_peft_model_state_dict(model, state_dict, adapter_name=model.active_adapter) + + # Invariant: `sd_options` is not None only for FSDP2 + if sd_options is not None: + from torch.distributed.checkpoint.state_dict import set_model_state_dict + + return set_model_state_dict(model, state_dict, options=sd_options) + else: + return model.load_state_dict(state_dict) + + +def _prepare_sd_options(fsdp_plugin): + sd_options = None + + # we use this only for FSDP2, as it requires torch >= 2.6.0 and this api requires torch >= 2.2.0 + if fsdp_plugin.fsdp_version == 2: + from torch.distributed.checkpoint.state_dict import StateDictOptions + from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType + + sd_options = StateDictOptions( + full_state_dict=fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT, + cpu_offload=getattr(fsdp_plugin.state_dict_config, "offload_to_cpu", False), + broadcast_from_rank0=getattr(fsdp_plugin.state_dict_config, "rank0_only", False), + ) + + return sd_options + + +def save_fsdp_model(fsdp_plugin, accelerator, model, output_dir, model_index=0, adapter_only=False): + # Note: We import here to reduce import time from general modules, and isolate outside dependencies + import torch.distributed.checkpoint as dist_cp + from torch.distributed.checkpoint.default_planner import DefaultSavePlanner + from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP + from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType + + os.makedirs(output_dir, exist_ok=True) + if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT: + # FSDP raises error when single GPU is used with `offload_to_cpu=True` for FULL_STATE_DICT + # so, only enable it when num_processes>1 + is_multi_process = accelerator.num_processes > 1 + fsdp_plugin.state_dict_config.offload_to_cpu = is_multi_process + fsdp_plugin.state_dict_config.rank0_only = is_multi_process + + ctx = ( + FSDP.state_dict_type( + model, fsdp_plugin.state_dict_type, fsdp_plugin.state_dict_config, fsdp_plugin.optim_state_dict_config + ) + if fsdp_plugin.fsdp_version == 1 + else nullcontext() + ) + sd_options = _prepare_sd_options(fsdp_plugin) + + with ctx: + state_dict = _get_model_state_dict(model, adapter_only=adapter_only, sd_options=sd_options) + if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT: + weights_name = f"{FSDP_MODEL_NAME}.bin" if model_index == 0 else f"{FSDP_MODEL_NAME}_{model_index}.bin" + output_model_file = os.path.join(output_dir, weights_name) + if accelerator.process_index == 0: + logger.info(f"Saving model to {output_model_file}") + torch.save(state_dict, output_model_file) + logger.info(f"Model saved to {output_model_file}") + # Invariant: `LOCAL_STATE_DICT` is never possible with `FSDP2` + elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT: + weights_name = ( + f"{FSDP_MODEL_NAME}_rank{accelerator.process_index}.bin" + if model_index == 0 + else f"{FSDP_MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin" + ) + output_model_file = os.path.join(output_dir, weights_name) + logger.info(f"Saving model to {output_model_file}") + torch.save(state_dict, output_model_file) + logger.info(f"Model saved to {output_model_file}") + elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT: + ckpt_dir = os.path.join(output_dir, f"{FSDP_MODEL_NAME}_{model_index}") + os.makedirs(ckpt_dir, exist_ok=True) + logger.info(f"Saving model to {ckpt_dir}") + state_dict = {"model": state_dict} + + dist_cp.save( + state_dict=state_dict, + storage_writer=dist_cp.FileSystemWriter(ckpt_dir), + planner=DefaultSavePlanner(), + ) + logger.info(f"Model saved to {ckpt_dir}") + + +def load_fsdp_model(fsdp_plugin, accelerator, model, input_dir, model_index=0, adapter_only=False): + # Note: We import here to reduce import time from general modules, and isolate outside dependencies + import torch.distributed.checkpoint as dist_cp + from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner + from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP + from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType + + accelerator.wait_for_everyone() + if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT: + # FSDP raises error when single GPU is used with `offload_to_cpu=True` for FULL_STATE_DICT + # so, only enable it when num_processes>1 + is_multi_process = accelerator.num_processes > 1 + fsdp_plugin.state_dict_config.offload_to_cpu = is_multi_process + fsdp_plugin.state_dict_config.rank0_only = is_multi_process + + ctx = ( + FSDP.state_dict_type( + model, fsdp_plugin.state_dict_type, fsdp_plugin.state_dict_config, fsdp_plugin.optim_state_dict_config + ) + if fsdp_plugin.fsdp_version == 1 + else nullcontext() + ) + sd_options = _prepare_sd_options(fsdp_plugin) + with ctx: + if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT: + if type(model) is not FSDP and accelerator.process_index != 0 and not accelerator.is_fsdp2: + if not fsdp_plugin.sync_module_states and fsdp_plugin.fsdp_version == 1: + raise ValueError( + "Set the `sync_module_states` flag to `True` so that model states are synced across processes when " + "initializing FSDP object" + ) + return + weights_name = f"{FSDP_MODEL_NAME}.bin" if model_index == 0 else f"{FSDP_MODEL_NAME}_{model_index}.bin" + input_model_file = os.path.join(input_dir, weights_name) + logger.info(f"Loading model from {input_model_file}") + # we want an empty state dict for FSDP2 as we use `broadcast_from_rank0` + load_model = not accelerator.is_fsdp2 or accelerator.is_main_process + if load_model: + state_dict = torch.load(input_model_file, weights_only=True) + else: + state_dict = {} + logger.info(f"Model loaded from {input_model_file}") + elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT: + weights_name = ( + f"{FSDP_MODEL_NAME}_rank{accelerator.process_index}.bin" + if model_index == 0 + else f"{FSDP_MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin" + ) + input_model_file = os.path.join(input_dir, weights_name) + logger.info(f"Loading model from {input_model_file}") + state_dict = torch.load(input_model_file, weights_only=True) + logger.info(f"Model loaded from {input_model_file}") + elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT: + ckpt_dir = ( + os.path.join(input_dir, f"{FSDP_MODEL_NAME}_{model_index}") + if f"{FSDP_MODEL_NAME}" not in input_dir + else input_dir + ) + logger.info(f"Loading model from {ckpt_dir}") + state_dict = {"model": _get_model_state_dict(model, adapter_only=adapter_only, sd_options=sd_options)} + dist_cp.load( + state_dict=state_dict, + storage_reader=dist_cp.FileSystemReader(ckpt_dir), + planner=DefaultLoadPlanner(), + ) + state_dict = state_dict["model"] + logger.info(f"Model loaded from {ckpt_dir}") + + load_result = _set_model_state_dict(model, state_dict, adapter_only=adapter_only, sd_options=sd_options) + return load_result + + +def save_fsdp_optimizer(fsdp_plugin, accelerator, optimizer, model, output_dir, optimizer_index=0): + # Note: We import here to reduce import time from general modules, and isolate outside dependencies + import torch.distributed.checkpoint as dist_cp + from torch.distributed.checkpoint.default_planner import DefaultSavePlanner + from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP + from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType + + os.makedirs(output_dir, exist_ok=True) + + ctx = ( + FSDP.state_dict_type( + model, fsdp_plugin.state_dict_type, fsdp_plugin.state_dict_config, fsdp_plugin.optim_state_dict_config + ) + if fsdp_plugin.fsdp_version == 1 + else nullcontext() + ) + + sd_options = _prepare_sd_options(fsdp_plugin) + + with ctx: + if fsdp_plugin.fsdp_version == 2: + from torch.distributed.checkpoint.state_dict import get_optimizer_state_dict + + optim_state = get_optimizer_state_dict(model, optimizer, options=sd_options) + else: + optim_state = FSDP.optim_state_dict(model, optimizer) + + if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT: + if accelerator.process_index == 0: + optim_state_name = ( + f"{OPTIMIZER_NAME}.bin" if optimizer_index == 0 else f"{OPTIMIZER_NAME}_{optimizer_index}.bin" + ) + output_optimizer_file = os.path.join(output_dir, optim_state_name) + logger.info(f"Saving Optimizer state to {output_optimizer_file}") + torch.save(optim_state, output_optimizer_file) + logger.info(f"Optimizer state saved in {output_optimizer_file}") + else: + ckpt_dir = os.path.join(output_dir, f"{OPTIMIZER_NAME}_{optimizer_index}") + os.makedirs(ckpt_dir, exist_ok=True) + logger.info(f"Saving Optimizer state to {ckpt_dir}") + dist_cp.save( + state_dict={"optimizer": optim_state}, + storage_writer=dist_cp.FileSystemWriter(ckpt_dir), + planner=DefaultSavePlanner(), + ) + logger.info(f"Optimizer state saved in {ckpt_dir}") + + +def load_fsdp_optimizer(fsdp_plugin, accelerator, optimizer, model, input_dir, optimizer_index=0, adapter_only=False): + # Note: We import here to reduce import time from general modules, and isolate outside dependencies + import torch.distributed.checkpoint as dist_cp + from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP + from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType + + accelerator.wait_for_everyone() + ctx = ( + FSDP.state_dict_type( + model, fsdp_plugin.state_dict_type, fsdp_plugin.state_dict_config, fsdp_plugin.optim_state_dict_config + ) + if fsdp_plugin.fsdp_version == 1 + else nullcontext() + ) + sd_options = _prepare_sd_options(fsdp_plugin) + with ctx: + if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT: + optim_state = None + if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only: + optimizer_name = ( + f"{OPTIMIZER_NAME}.bin" if optimizer_index == 0 else f"{OPTIMIZER_NAME}_{optimizer_index}.bin" + ) + input_optimizer_file = os.path.join(input_dir, optimizer_name) + logger.info(f"Loading Optimizer state from {input_optimizer_file}") + optim_state = torch.load(input_optimizer_file, weights_only=True) + logger.info(f"Optimizer state loaded from {input_optimizer_file}") + else: + ckpt_dir = ( + os.path.join(input_dir, f"{OPTIMIZER_NAME}_{optimizer_index}") + if f"{OPTIMIZER_NAME}" not in input_dir + else input_dir + ) + logger.info(f"Loading Optimizer from {ckpt_dir}") + if fsdp_plugin.fsdp_version == 2: + from torch.distributed.checkpoint.state_dict import get_optimizer_state_dict + + optim_state = get_optimizer_state_dict(model, optimizer, options=sd_options) + else: + optim_state = FSDP.optim_state_dict(model, optimizer) + optim_state = {"optimizer": optim_state} + dist_cp.load( + optim_state, + checkpoint_id=ckpt_dir, + storage_reader=dist_cp.FileSystemReader(ckpt_dir), + ) + optim_state = optim_state["optimizer"] + logger.info(f"Optimizer loaded from {ckpt_dir}") + + if fsdp_plugin.fsdp_version == 1: + flattened_osd = FSDP.optim_state_dict_to_load(model=model, optim=optimizer, optim_state_dict=optim_state) + optimizer.load_state_dict(flattened_osd) + else: + from torch.distributed.checkpoint.state_dict import set_optimizer_state_dict + + set_optimizer_state_dict(model, optimizer, optim_state, options=sd_options) + + +def _distributed_checkpoint_to_merged_weights(checkpoint_dir: str, save_path: str, safe_serialization: bool = True): + """ + Passthrough to `torch.distributed.checkpoint.format_utils.dcp_to_torch_save` + + Will save under `save_path` as either `model.safetensors` or `pytorch_model.bin`. + """ + # Note: We import here to reduce import time from general modules, and isolate outside dependencies + import torch.distributed.checkpoint as dist_cp + import torch.distributed.checkpoint.format_utils as dist_cp_format_utils + + state_dict = {} + save_path = Path(save_path) + save_path.mkdir(exist_ok=True) + dist_cp_format_utils._load_state_dict( + state_dict, + storage_reader=dist_cp.FileSystemReader(checkpoint_dir), + planner=dist_cp_format_utils._EmptyStateDictLoadPlanner(), + no_dist=True, + ) + save_path = save_path / SAFE_WEIGHTS_NAME if safe_serialization else save_path / WEIGHTS_NAME + + # To handle if state is a dict like {model: {...}} + if len(state_dict.keys()) == 1: + state_dict = state_dict[list(state_dict)[0]] + save(state_dict, save_path, safe_serialization=safe_serialization) + return save_path + + +def merge_fsdp_weights( + checkpoint_dir: str, output_path: str, safe_serialization: bool = True, remove_checkpoint_dir: bool = False +): + """ + Merge the weights from sharded FSDP model checkpoints into a single combined checkpoint. Should be used if + `SHARDED_STATE_DICT` was used for the model. Weights will be saved to `{output_path}/model.safetensors` if + `safe_serialization` else `pytorch_model.bin`. + + Note: this is a CPU-bound process. + + Args: + checkpoint_dir (`str`): + The directory containing the FSDP checkpoints (can be either the model or optimizer). + output_path (`str`): + The path to save the merged checkpoint. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the merged weights with safetensors (recommended). + remove_checkpoint_dir (`bool`, *optional*, defaults to `False`): + Whether to remove the checkpoint directory after merging. + """ + checkpoint_dir = Path(checkpoint_dir) + from accelerate.state import PartialState + + if not is_torch_version(">=", "2.3.0"): + raise ValueError("`merge_fsdp_weights` requires PyTorch >= 2.3.0`") + + # Verify that the checkpoint directory exists + if not checkpoint_dir.exists(): + model_path_exists = (checkpoint_dir / "pytorch_model_fsdp_0").exists() + optimizer_path_exists = (checkpoint_dir / "optimizer_0").exists() + err = f"Tried to load from {checkpoint_dir} but couldn't find a valid metadata file." + if model_path_exists and optimizer_path_exists: + err += " However, potential model and optimizer checkpoint directories exist." + err += f"Please pass in either {checkpoint_dir}/pytorch_model_fsdp_0 or {checkpoint_dir}/optimizer_0" + err += "instead." + elif model_path_exists: + err += " However, a potential model checkpoint directory exists." + err += f"Please try passing in {checkpoint_dir}/pytorch_model_fsdp_0 instead." + elif optimizer_path_exists: + err += " However, a potential optimizer checkpoint directory exists." + err += f"Please try passing in {checkpoint_dir}/optimizer_0 instead." + raise ValueError(err) + + # To setup `save` to work + state = PartialState() + if state.is_main_process: + logger.info(f"Merging FSDP weights from {checkpoint_dir}") + save_path = _distributed_checkpoint_to_merged_weights(checkpoint_dir, output_path, safe_serialization) + logger.info(f"Successfully merged FSDP weights and saved to {save_path}") + if remove_checkpoint_dir: + logger.info(f"Removing old checkpoint directory {checkpoint_dir}") + shutil.rmtree(checkpoint_dir) + state.wait_for_everyone() + + +def ensure_weights_retied(param_init_fn, model: torch.nn.Module, device: torch.device): + _tied_names = getattr(model, "_tied_weights_keys", None) + if not _tied_names: + # if no tied names just passthrough + return param_init_fn + + # get map of parameter instances to params. + # - needed for replacement later + _tied_params = {} + for name in _tied_names: + name = name.split(".") + name, param_name = ".".join(name[:-1]), name[-1] + mod = model.get_submodule(name) + param = getattr(mod, param_name) + + _tied_params[id(param)] = None # placeholder for the param first + + # build param_init_fn for the case with tied params + def param_init_fn_tied_param(module: torch.nn.Module): + # track which params to tie + # - usually only 1, but for completeness consider > 1 + params_to_tie = defaultdict(list) + for n, param in module.named_parameters(recurse=False): + if id(param) in _tied_params: + params_to_tie[id(param)].append(n) + + # call the param init fn, which potentially re-allocates the + # parameters + module = param_init_fn(module) + + # search the parameters again and tie them up again + for id_key, _param_names in params_to_tie.items(): + for param_name in _param_names: + param = _tied_params[id_key] + if param is None: + # everything will be tied to the first time the + # param is observed + _tied_params[id_key] = getattr(module, param_name) + else: + setattr(module, param_name, param) # tie + + return module + + return param_init_fn_tied_param + + +def fsdp2_load_full_state_dict(accelerator, model: torch.nn.Module, full_sd: dict, cpu_offload: bool = False): + """ + Loads the full state dict (could be only on rank 0) into the sharded model. This is done by broadcasting the + parameters from rank 0 to all other ranks. This function modifies the model in-place. + + Args: + accelerator (`Accelerator`): The accelerator instance + model (`torch.nn.Module`): + The model to load the state dict into, expected to be on meta device or a VRAM spike can occur + full_sd (`dict`): The full state dict to load, can only be on rank 0 + cpu_offload (`bool`, defaults to `False`): + If True, move sharded parameters to CPU after distribution. Required when FSDP CPU offloading is enabled. + """ + import torch.distributed as dist + from torch.distributed.tensor import DTensor, distribute_tensor + + # Model was previously copied to meta device + meta_sharded_sd = model.state_dict() + sharded_sd = {} + + # Rank 0 distributes the full state dict to other ranks + def _infer_parameter_dtype(model, param_name, empty_param): + try: + old_param = model.get_parameter_or_buffer(param_name) + except AttributeError: + # Need this for LORA, as there some params are not *parameters* of sorts + base_param_name, local_param_name = param_name.rsplit(".", 1) + submodule = model.get_submodule(base_param_name) + old_param = getattr(submodule, local_param_name) + + is_torch_e4m3fn_available = hasattr(torch, "float8_e4m3fn") + casting_dtype = None + is_param_float8_e4m3fn = is_torch_e4m3fn_available and empty_param.dtype == torch.float8_e4m3fn + + if empty_param.dtype.is_floating_point and not is_param_float8_e4m3fn: + casting_dtype = old_param.dtype + + return old_param is not None and old_param.is_contiguous(), casting_dtype + + def _cast_and_contiguous(tensor, to_contiguous, dtype): + if dtype is not None: + tensor = tensor.to(dtype=dtype) + if to_contiguous: + tensor = tensor.contiguous() + return tensor + + if accelerator.is_main_process: + for param_name, sharded_param in meta_sharded_sd.items(): + if param_name not in full_sd: + raise KeyError( + f"Parameter '{param_name}' found in sharded model state dict but missing from full state dict. " + f"Full state dict has {len(full_sd)} keys, sharded has {len(meta_sharded_sd)} keys." + ) + full_param = full_sd[param_name] + device_mesh = sharded_param.device_mesh + full_param = full_param.detach().to(device_mesh.device_type) + if isinstance(full_param, DTensor): + # dist.broadcast() only supports torch.Tensor. + # After prepare_tp(), model parameters may become DTensor. + # To broadcast such a parameter, convert it to a local tensor first. + full_param = full_param.to_local() + dist.broadcast(full_param, src=0, group=dist.group.WORLD) + sharded_tensor = distribute_tensor(full_param, device_mesh, sharded_param.placements) + to_contiguous, casting_dtype = _infer_parameter_dtype( + model, + param_name, + full_param, + ) + sharded_tensor = _cast_and_contiguous(sharded_tensor, to_contiguous, casting_dtype) + # When CPU offloading is enabled, FSDP2's lazy_init expects parameters on CPU + if cpu_offload: + sharded_tensor = sharded_tensor.to("cpu") + sharded_sd[param_name] = sharded_tensor + # We need this else to have a matching `broadcast` for all of the ranks, else we deadlock + else: + for param_name, sharded_param in meta_sharded_sd.items(): + device_mesh = sharded_param.device_mesh + full_tensor = torch.empty(sharded_param.size(), device=device_mesh.device_type, dtype=sharded_param.dtype) + dist.broadcast(full_tensor, src=0, group=dist.group.WORLD) + sharded_tensor = distribute_tensor(full_tensor, device_mesh, sharded_param.placements) + to_contiguous, casting_dtype = _infer_parameter_dtype( + model, + param_name, + full_tensor, + ) + sharded_tensor = _cast_and_contiguous(sharded_tensor, to_contiguous, casting_dtype) + # When CPU offloading is enabled, FSDP2's lazy_init expects parameters on CPU + if cpu_offload: + sharded_tensor = sharded_tensor.to("cpu") + sharded_sd[param_name] = sharded_tensor + + # we set `assign=True` because our params are on meta device + model.load_state_dict(sharded_sd, assign=True) + return model + + +def fsdp2_switch_optimizer_parameters(optimizer: torch.optim.Optimizer, mapping: dict): + """ + Switches the parameters of the optimizer to new ones (sharded parameters in usual case). This function modifies the + optimizer in-place. + + Args: + optimizer (`torch.optim.Optimizer`): Optimizer instance which contains the original model parameters + mapping (`dict`): Mapping from the original parameter (specified by `data_ptr`) to the sharded parameter + + Raises: + KeyError: + If a parameter in the optimizer couldn't be switched to its sharded version. This should never happen and + indicates a bug. If we kept the original params instead of raising, the training wouldn't be numerically + correct and weights wouldn't get updated. + """ + from torch.distributed.tensor import DTensor + + accessor_mapping = {} + + accessor_mapping[DTensor] = "_local_tensor" + try: + for param_group in optimizer.param_groups: + param_group["params"] = [mapping[p.data_ptr] for p in param_group["params"]] + except KeyError: + # This shouldn't ever happen, but we want to fail here else training wouldn't be numerically correct + # This basically means that we're missing a mapping from the original parameter to the sharded parameter + raise KeyError( + "A parameter in the optimizer couldn't be switched to its sharded version. This breaks the training. Please raise an issue on GitHub." + ) + + +def fsdp2_apply_ac(accelerator, model: torch.nn.Module): + """ + Applies the activation checkpointing to the model. + + Args: + accelerator (`Accelerator`): The accelerator instance + model (`torch.nn.Module`): The model to apply the activation checkpointing to + + Returns: + `torch.nn.Module`: The model with the activation checkpointing applied + """ + + from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import ( + checkpoint_wrapper, + ) + + auto_wrap_policy_func = fsdp2_prepare_auto_wrap_policy(accelerator.state.fsdp_plugin, model) + + for layer_name, layer in get_module_children_bottom_up(model, return_fqns=True)[:-1]: + if len(layer_name.split(".")) > 1: + parent_name, child_name = layer_name.rsplit(".", 1) + else: + parent_name = None + child_name = layer_name + + parent_module = model.get_submodule(parent_name) if parent_name else model + if auto_wrap_policy_func(parent_module): + layer = checkpoint_wrapper(layer, preserve_rng_state=False) + parent_module.register_module(child_name, layer) + + return model + + +def _find_final_norm(model: torch.nn.Module) -> torch.nn.Module | None: + """Find the final normalization layer before the output head. + + The final norm is conventionally a direct child of the base model (e.g. `model.norm` + for Llama, `transformer.ln_f` for GPT-2), so we only scan the base model's direct + children. Returns the last norm found there, or None. + """ + base_prefix = getattr(model, "base_model_prefix", "") + base_model = getattr(model, base_prefix, None) if base_prefix else model + if not isinstance(base_model, torch.nn.Module): + return None + final_norm = None + for _, module in base_model.named_children(): + if "Norm" in type(module).__name__: + final_norm = module + return final_norm + + +def fsdp2_prepare_model(accelerator, model: torch.nn.Module) -> torch.nn.Module: + """Prepares the model for FSDP2 in-place. Also returns the model to avoid misuse of the original model. + + Args: + accelerator (`Accelerator`): The accelerator instance + model (`torch.nn.Module`): The model to prepare + + Returns: + `torch.nn.Module`: Prepared model + """ + from torch.distributed.fsdp import FSDPModule, MixedPrecisionPolicy, fully_shard + + is_type_fsdp = isinstance(model, FSDPModule) or ( + is_compiled_module(model) and isinstance(model._orig_mod, FSDPModule) + ) + if is_type_fsdp: + return model + + fsdp2_plugin = accelerator.state.fsdp_plugin + + fsdp2_plugin.set_auto_wrap_policy(model) + + mesh = getattr(accelerator, "torch_device_mesh", None) + + fsdp2_kwargs = { + "reshard_after_forward": fsdp2_plugin.reshard_after_forward, + "offload_policy": fsdp2_plugin.cpu_offload, + # `fully_shard` does not accept `None` in case of `MixedPrecisionPolicy` + "mp_policy": fsdp2_plugin.mixed_precision_policy or MixedPrecisionPolicy(), + "mesh": mesh[tuple(accelerator.parallelism_config.fsdp_dim_names)] if mesh is not None else None, + } + + # `ignored_params` is only supported in torch >= 2.7.0 + if is_torch_version(">=", "2.7.0") and fsdp2_plugin.ignored_modules is not None: + fsdp2_kwargs["ignored_params"] = get_parameters_from_modules( + fsdp2_plugin.ignored_modules, model, accelerator.device + ) + + model_has_params4bit = False + incompatible_params4bit = set() + for name, param in model.named_parameters(): + # this is a temporary fix whereby loading models with bnb params cannot be moved from + # GPU to a meta device due with FSDP2 because torch operations don't return the original class type + # bypassing the move to meta will still cause the VRAM spike, but at least it still will load + if param.__class__.__name__ == "Params4bit": + model_has_params4bit = True + # Exclude non-floating frozen Params4bit from FSDP sharding. + # Default uint8 quant_storage cannot survive fully_shard's DTensor conversion. + if (not param.requires_grad) and (not param.is_floating_point()) and (not param.is_complex()): + incompatible_params4bit.add(param) + + if incompatible_params4bit and is_torch_version(">=", "2.7.0"): + ignored = set(fsdp2_kwargs.get("ignored_params", set())) + fsdp2_kwargs["ignored_params"] = ignored | incompatible_params4bit + if accelerator.is_main_process: + warnings.warn( + f"Found {len(incompatible_params4bit)} non-floating frozen Params4bit. " + "Excluding from FSDP2 sharding to prevent quant_state corruption." + "To enable memory-efficient sharding of 4-bit weights, set" + "bnb_4bit_quant_storage to a floating dtype (e.g. bf16)." + ) + + # FSDP2 requires uniform orig_dtype among trainable params in each group. + # Upcast to fp32 master weights; MixedPrecisionPolicy.param_dtype handles compute cast. + if accelerator.mixed_precision != "no" and not model_has_params4bit: + upcasted_params = [] + for name, param in model.named_parameters(): + if param.requires_grad and param.dtype != torch.float32: + upcasted_params.append(name) + param.data = param.data.to(torch.float32) + if accelerator.is_main_process and upcasted_params: + logger.info( + "FSDP upcast of low precision parameters to fp32 (since mixed_precision != 'no') may affect the precision of model checkpoints. " + f"This effects {len(upcasted_params)} parameters: {upcasted_params}..." + ) + + # Capture after upcast so dtypes match what `fully_shard` will produce. + original_sd = model.state_dict() + + if fsdp2_plugin.cpu_ram_efficient_loading and not model_has_params4bit: + # Context: `fully_shard` moves the model to GPU if it was on CPU, however it can also be on `meta` and then it stays there even after `fully_shard` + # For this reason, we need to move the model to `meta` device, as then sharding happens on `meta` device + # If we kept the model on CPU (`cpu_ram_efficient_loading` has model be on CPU on all ranks, though non-main ranks only have `torch.empty`), `fully_shard` would move it to GPU + # Afterwards, when we call `fsdp2_load_full_state_dict`, us creating the state_dict would result into briefly having two copies of model state_dict on the GPU -> VRAM spike + + # We need to keep the original non-persistent buffers, as those MAY not be in the state_dict, resulting in them staying on meta device + # Also, these buffers aren't getting sharded by default + # We get the FQNs of all non-persistent buffers, to re-register them after + non_persistent_buffer_fqns = get_non_persistent_buffers(model, recurse=True, fqns=True) + original_non_persistent_buffers = copy.deepcopy( + {k: v for k, v in model.named_buffers() if k in non_persistent_buffer_fqns} + ) + # We move the model to meta device, as then sharding happens on meta device + model = model.to(torch.device("meta")) + # We need to re-tie the weights, not exactly sure why, but if we don't do this, reference to `lm_head/embed_tokens` stay hanging -> more VRAM usage + # We assume `transformers` models have a `tie_weights` method if they support it + if hasattr(model, "tie_weights"): + model.tie_weights() + + auto_wrap_policy_func = fsdp2_prepare_auto_wrap_policy(fsdp2_plugin, model) + if auto_wrap_policy_func is not None: + # We skip the model itself, as that one is always wrapped + for module in get_module_children_bottom_up(model)[:-1]: + if auto_wrap_policy_func(module) and not isinstance(module, FSDPModule): + fully_shard(module, **fsdp2_kwargs) + + # Carve embed + tail=[final_norm, lm_head] into their own units. Tail gets `False` + # (forward-end / backward-start uses cluster, re-gather would be unhideable). + input_embed = getattr(model, "get_input_embeddings", lambda: None)() + output_embed = getattr(model, "get_output_embeddings", lambda: None)() + input_weight = getattr(input_embed, "weight", None) + output_weight = getattr(output_embed, "weight", None) + is_weights_tied = input_weight is not None and input_weight is output_weight + + if not is_weights_tied and input_embed is not None and not isinstance(input_embed, FSDPModule): + fully_shard(input_embed, **fsdp2_kwargs) + + final_norm = _find_final_norm(model) + # Tied case uses `input_embed` so the pre-forward hook fires at forward-start + # (embed's use of the shared tensor), not at forward-end with `output_embed`. + tail_embed = input_embed if is_weights_tied else output_embed + tail = [m for m in (final_norm, tail_embed) if m is not None and not isinstance(m, FSDPModule)] + if tail: + fully_shard(tail, **{**fsdp2_kwargs, "reshard_after_forward": False}) + + if not isinstance(model, FSDPModule): + # Defer to PyTorch's `reshard_after_forward=None` heuristic, which resolves to `False` + # for the root. Avoids an unhideable pre-backward all-gather of the root's leftovers. + root_kwargs = {k: v for k, v in fsdp2_kwargs.items() if k != "reshard_after_forward"} + fully_shard(model, **root_kwargs) + + if fsdp2_plugin.cpu_ram_efficient_loading and not model_has_params4bit: + # If `cpu_ram_efficient_loading` is enabled, only rank 0 loads the weights + # Other ranks have an empty model on `meta` device, so we need to distribute the weights properly + # When CPU offloading is enabled, parameters need to stay on CPU after distribution + from torch.distributed.fsdp import CPUOffloadPolicy + + fsdp2_load_full_state_dict( + accelerator, model, original_sd, cpu_offload=isinstance(fsdp2_plugin.cpu_offload, CPUOffloadPolicy) + ) + + if fsdp2_plugin.cpu_ram_efficient_loading and not model_has_params4bit: + # We re-register the buffers, as they may not be in the state_dict + for fqn, buffer_tensor in original_non_persistent_buffers.items(): + buffer_tensor = buffer_tensor.to(accelerator.device) + + if "." in fqn: + parent_fqn, local_buffer_name = fqn.rsplit(".", 1) + parent_module = model.get_submodule(parent_fqn) + else: + local_buffer_name = fqn + parent_module = model + + parent_module.register_buffer(local_buffer_name, buffer_tensor, persistent=False) + + # We need to tie the weights again, as call to `load_full_state_dict` breaks the tie + # Needs to be called both here and above + # removing this call makes the have slightly different loss + # removing the call above leads to extra memory usage as explained in the comment above + if hasattr(model, "tie_weights"): + model.tie_weights() + + return model + + +def fsdp2_prepare_auto_wrap_policy(fsdp2_plugin, model: torch.nn.Module) -> Callable[[torch.nn.Module], bool] | None: + """Prepares the auto wrap policy based on its type, done to mimic the behaviour of FSDP1 auto wrap policy. + + Args: + fsdp2_plugin (`FullyShardedDataParallelPlugin`): + Instance of `FullyShardedDataParallelPlugin` containing the configuration options + model (`torch.nn.Module`): + The model to wrap + + Returns: + `Callable[[torch.nn.Module], bool] | None`: + The auto wrap policy function to be applied to the model or `None` + """ + from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy, transformer_auto_wrap_policy + + fn = fsdp2_plugin.auto_wrap_policy + + if isinstance(fn, functools.partial): + fn = fn.func + + if fn is transformer_auto_wrap_policy: + no_split_modules = getattr(model, "_no_split_modules", None) + if no_split_modules is None: + no_split_modules = [] + transformer_cls_names_to_wrap = list(no_split_modules) + if fsdp2_plugin.transformer_cls_names_to_wrap is not None: + transformer_cls_names_to_wrap = fsdp2_plugin.transformer_cls_names_to_wrap + transformer_cls_to_wrap = set() + + for layer_class in transformer_cls_names_to_wrap: + transformer_cls = get_module_class_from_name(model, layer_class) + if transformer_cls is None: + raise ValueError(f"Could not find the transformer layer class {layer_class} in the model.") + transformer_cls_to_wrap.add(transformer_cls) + + def policy(module: torch.nn.Module) -> bool: + if not transformer_cls_to_wrap: + return False + return isinstance(module, tuple(transformer_cls_to_wrap)) + + elif fn is size_based_auto_wrap_policy: + + def policy(module: torch.nn.Module) -> bool: + module_num_params = sum(p.numel() for p in module.parameters()) + return module_num_params > fsdp2_plugin.min_num_params + else: + return None + + return policy + + +def get_fsdp2_grad_scaler(**kwargs): + """ + Returns a `GradScaler` for FSDP2, as the current implementation of `get_grad_scaler` doesn't accept other args. We + need this as current `get_grad_scaler` accepts only `distributed_type` as arg, which doesn't differentiate between + FSDP1 and FSDP2 + """ + from torch.amp.grad_scaler import GradScaler + + return GradScaler(**kwargs) + + +def fsdp2_canonicalize_names(named_params: dict) -> dict: + """Removes parameter name modifiers in order to map them back to their original names. + + See huggingface/accelerate#3554 for more context. + + Args: + named_params (`dict`): The named parameters dictionary to canonicalize. + + Returns: + `dict`: The canonicalized named parameters dictionary + """ + named_params = {k.replace("._checkpoint_wrapped_module", ""): v for k, v in named_params.items()} + named_params = { + k.replace("_orig_mod.", "") if k.startswith("_orig_mod.") else k: v for k, v in named_params.items() + } + named_params = {k.replace("._orig_mod", ""): v for k, v in named_params.items()} + return named_params + + +def get_parameters_from_modules( + modules: Union[Iterable[torch.nn.Module], str], model, device +) -> set[torch.nn.Parameter]: + """Converts modules to parameters where modules can be a string or list of torch.nn.Module + + Args: + modules (`Union[Iterable[torch.nn.Module], str]`): List of modules + + Returns: + `set[torch.nn.Parameter]`: List of parameters + """ + if modules is None: + return set() + parameters = [] + # code taken from accelerate while preparing kwargs for FSDP + if isinstance(modules, str): + reg = re.compile(modules) + mapped_modules = [] + for name, module in model.named_modules(): + if reg.fullmatch(name): + module.to(device) + mapped_modules.append(module) + modules = mapped_modules + for module in modules: + parameters.extend(list(module.parameters())) + return set(parameters) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/imports.py b/kbidu_env/Lib/site-packages/accelerate/utils/imports.py new file mode 100644 index 0000000000000000000000000000000000000000..32932c2a1c1099684d53a3aa10d842a6f7f5266c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/imports.py @@ -0,0 +1,544 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import importlib.metadata +import os +import sys +import warnings +from functools import lru_cache, wraps + +import torch +from packaging import version +from packaging.version import parse + +from .environment import parse_flag_from_env, patch_environment, str_to_bool +from .versions import compare_versions, is_torch_version + + +# Try to run Torch native job in an environment with TorchXLA installed by setting this value to 0. +USE_TORCH_XLA = parse_flag_from_env("USE_TORCH_XLA", default=True) + +_torch_xla_available = False +if USE_TORCH_XLA: + try: + import torch_xla.core.xla_model as xm # noqa: F401 + import torch_xla.runtime + + _torch_xla_available = True + except ImportError: + pass + +# Keep it for is_tpu_available. It will be removed along with is_tpu_available. +_tpu_available = _torch_xla_available + +# Cache this result has it's a C FFI call which can be pretty time-consuming +_torch_distributed_available = torch.distributed.is_available() + + +def _is_package_available(pkg_name, metadata_name=None): + # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version + package_exists = importlib.util.find_spec(pkg_name) is not None + if package_exists: + try: + # Some libraries have different names in the metadata + _ = importlib.metadata.metadata(pkg_name if metadata_name is None else metadata_name) + return True + except importlib.metadata.PackageNotFoundError: + return False + + +def is_torch_distributed_available() -> bool: + return _torch_distributed_available + + +def is_xccl_available(): + if is_torch_version(">=", "2.7.0"): + return torch.distributed.distributed_c10d.is_xccl_available() + return False + + +def is_import_timer_available(): + return _is_package_available("import_timer") + + +def is_pynvml_available(): + return _is_package_available("pynvml") or _is_package_available("pynvml", "nvidia-ml-py") + + +def is_amdsmi_available(): + return _is_package_available("amdsmi") + + +def is_rocm_available(): + return torch.version.hip is not None and torch.cuda.is_available() + + +def is_pytest_available(): + return _is_package_available("pytest") + + +def is_msamp_available(): + return _is_package_available("msamp", "ms-amp") + + +def is_schedulefree_available(): + return _is_package_available("schedulefree") + + +def is_transformer_engine_available(): + if is_hpu_available(): + return _is_package_available("intel_transformer_engine", "intel-transformer-engine") + else: + return _is_package_available("transformer_engine", "transformer-engine") + + +def is_transformer_engine_mxfp8_available(): + if _is_package_available("transformer_engine", "transformer-engine"): + from transformer_engine.pytorch.fp8 import check_mxfp8_support + + return check_mxfp8_support()[0] + return False + + +def is_lomo_available(): + return _is_package_available("lomo_optim") + + +def is_cuda_available(): + """ + Checks if `cuda` is available via an `nvml-based` check which won't trigger the drivers and leave cuda + uninitialized. + """ + with patch_environment(PYTORCH_NVML_BASED_CUDA_CHECK="1"): + available = torch.cuda.is_available() + + return available + + +@lru_cache +def is_torch_xla_available(check_is_tpu=False, check_is_gpu=False): + """ + Check if `torch_xla` is available. To train a native pytorch job in an environment with torch xla installed, set + the USE_TORCH_XLA to false. + """ + assert not (check_is_tpu and check_is_gpu), "The check_is_tpu and check_is_gpu cannot both be true." + + if not _torch_xla_available: + return False + elif check_is_gpu: + return torch_xla.runtime.device_type() in ["GPU", "CUDA"] + elif check_is_tpu: + return torch_xla.runtime.device_type() == "TPU" + + return True + + +def is_torchao_available(): + package_exists = _is_package_available("torchao") + if package_exists: + torchao_version = version.parse(importlib.metadata.version("torchao")) + return compare_versions(torchao_version, ">=", "0.6.1") + return False + + +def is_deepspeed_available(): + return _is_package_available("deepspeed") + + +def is_pippy_available(): + return is_torch_version(">=", "2.4.0") + + +def is_bf16_available(ignore_tpu=False): + "Checks if bf16 is supported, optionally ignoring the TPU" + if is_torch_xla_available(check_is_tpu=True): + return not ignore_tpu + if is_cuda_available(): + return torch.cuda.is_bf16_supported() + if is_mlu_available(): + return torch.mlu.is_bf16_supported() + if is_xpu_available(): + return torch.xpu.is_bf16_supported() + if is_mps_available(): + return torch.backends.mps.is_macos_or_newer(14, 0) + return True + + +def is_fp16_available(): + "Checks if fp16 is supported" + if is_habana_gaudi1(): + return False + + return True + + +def is_fp8_available(): + "Checks if fp8 is supported" + return is_msamp_available() or is_transformer_engine_available() or is_torchao_available() + + +def is_4bit_bnb_available(): + package_exists = _is_package_available("bitsandbytes") + if package_exists: + bnb_version = version.parse(importlib.metadata.version("bitsandbytes")) + return compare_versions(bnb_version, ">=", "0.39.0") + return False + + +def is_8bit_bnb_available(): + package_exists = _is_package_available("bitsandbytes") + if package_exists: + bnb_version = version.parse(importlib.metadata.version("bitsandbytes")) + return compare_versions(bnb_version, ">=", "0.37.2") + return False + + +def is_bnb_available(min_version=None): + package_exists = _is_package_available("bitsandbytes") + if package_exists and min_version is not None: + bnb_version = version.parse(importlib.metadata.version("bitsandbytes")) + return compare_versions(bnb_version, ">=", min_version) + else: + return package_exists + + +def is_bitsandbytes_multi_backend_available(): + if not is_bnb_available(): + return False + import bitsandbytes as bnb + + return "multi_backend" in getattr(bnb, "features", set()) + + +def is_torchvision_available(): + return _is_package_available("torchvision") + + +def is_megatron_lm_available(): + if str_to_bool(os.environ.get("ACCELERATE_USE_MEGATRON_LM", "False")) == 1: + if importlib.util.find_spec("megatron") is not None: + try: + megatron_version = parse(importlib.metadata.version("megatron-core")) + if compare_versions(megatron_version, ">=", "0.8.0"): + return importlib.util.find_spec(".training", "megatron") + except Exception as e: + warnings.warn(f"Parse Megatron version failed. Exception:{e}") + return False + + +def is_transformers_available(): + return _is_package_available("transformers") + + +def is_datasets_available(): + return _is_package_available("datasets") + + +def is_peft_available(): + return _is_package_available("peft") + + +def is_timm_available(): + return _is_package_available("timm") + + +def is_triton_available(): + if is_xpu_available(): + return _is_package_available("triton", "triton-xpu") + return _is_package_available("triton") + + +def is_aim_available(): + package_exists = _is_package_available("aim") + if package_exists: + aim_version = version.parse(importlib.metadata.version("aim")) + return compare_versions(aim_version, "<", "4.0.0") + return False + + +def is_tensorboard_available(): + return _is_package_available("tensorboard") or _is_package_available("tensorboardX") + + +def is_wandb_available(): + return _is_package_available("wandb") + + +def is_comet_ml_available(): + return _is_package_available("comet_ml") + + +def is_swanlab_available(): + return _is_package_available("swanlab") + + +def is_trackio_available(): + return sys.version_info >= (3, 10) and _is_package_available("trackio") + + +def is_boto3_available(): + return _is_package_available("boto3") + + +def is_rich_available(): + if _is_package_available("rich"): + return parse_flag_from_env("ACCELERATE_ENABLE_RICH", False) + return False + + +def is_sagemaker_available(): + return _is_package_available("sagemaker") + + +def is_tqdm_available(): + return _is_package_available("tqdm") + + +def is_clearml_available(): + return _is_package_available("clearml") + + +def is_pandas_available(): + return _is_package_available("pandas") + + +def is_matplotlib_available(): + return _is_package_available("matplotlib") + + +def is_mlflow_available(): + if _is_package_available("mlflow"): + return True + + if importlib.util.find_spec("mlflow") is not None: + try: + _ = importlib.metadata.metadata("mlflow-skinny") + return True + except importlib.metadata.PackageNotFoundError: + return False + return False + + +def is_mps_available(min_version="1.12"): + "Checks if MPS device is available. The minimum version required is 1.12." + # With torch 1.12, you can use torch.backends.mps + # With torch 2.0.0, you can use torch.mps + return is_torch_version(">=", min_version) and torch.backends.mps.is_available() and torch.backends.mps.is_built() + + +@lru_cache +def is_mlu_available(check_device=False): + """ + Checks if `mlu` is available via an `cndev-based` check which won't trigger the drivers and leave mlu + uninitialized. + """ + if importlib.util.find_spec("torch_mlu") is None: + return False + + import torch_mlu # noqa: F401 + + with patch_environment(PYTORCH_CNDEV_BASED_MLU_CHECK="1"): + available = torch.mlu.is_available() + + return available + + +@lru_cache +def is_musa_available(check_device=False): + "Checks if `torch_musa` is installed and potentially if a MUSA is in the environment" + if importlib.util.find_spec("torch_musa") is None: + return False + + import torch_musa # noqa: F401 + + if check_device: + try: + # Will raise a RuntimeError if no MUSA is found + _ = torch.musa.device_count() + return torch.musa.is_available() + except RuntimeError: + return False + return hasattr(torch, "musa") and torch.musa.is_available() + + +@lru_cache +def is_npu_available(check_device=False): + "Checks if `torch_npu` is installed and potentially if a NPU is in the environment" + if importlib.util.find_spec("torch_npu") is None: + return False + + # NOTE: importing torch_npu may raise error in some envs + # e.g. inside cpu-only container with torch_npu installed + try: + import torch_npu # noqa: F401 + except Exception: + return False + + if check_device: + try: + # Will raise a RuntimeError if no NPU is found + _ = torch.npu.device_count() + return torch.npu.is_available() + except RuntimeError: + return False + return hasattr(torch, "npu") and torch.npu.is_available() + + +@lru_cache +def is_sdaa_available(check_device=False): + "Checks if `torch_sdaa` is installed and potentially if a SDAA is in the environment" + if importlib.util.find_spec("torch_sdaa") is None: + return False + + import torch_sdaa # noqa: F401 + + if check_device: + try: + # Will raise a RuntimeError if no NPU is found + _ = torch.sdaa.device_count() + return torch.sdaa.is_available() + except RuntimeError: + return False + return hasattr(torch, "sdaa") and torch.sdaa.is_available() + + +@lru_cache +def is_hpu_available(init_hccl=False): + "Checks if `torch.hpu` is installed and potentially if a HPU is in the environment" + if ( + importlib.util.find_spec("habana_frameworks") is None + or importlib.util.find_spec("habana_frameworks.torch") is None + ): + return False + + import habana_frameworks.torch # noqa: F401 + + if init_hccl: + import habana_frameworks.torch.distributed.hccl as hccl # noqa: F401 + + return hasattr(torch, "hpu") and torch.hpu.is_available() + + +def is_habana_gaudi1(): + if is_hpu_available(): + import habana_frameworks.torch.utils.experimental as htexp # noqa: F401 + + if htexp._get_device_type() == htexp.synDeviceType.synDeviceGaudi: + return True + + return False + + +@lru_cache +def is_xpu_available(check_device=False): + """ + Checks if XPU acceleration is available via stock PyTorch (>=2.7) and + potentially if a XPU is in the environment + """ + + if is_torch_version("<=", "2.6"): + return False + + if check_device: + try: + # Will raise a RuntimeError if no XPU is found + _ = torch.xpu.device_count() + return torch.xpu.is_available() + except RuntimeError: + return False + return hasattr(torch, "xpu") and torch.xpu.is_available() + + +@lru_cache +def is_neuron_available(check_device=False): + if importlib.util.find_spec("torch_neuronx") is None: + return False + + if check_device: + try: + import torch_neuronx # noqa: F401 + + # Will raise a RuntimeError if no Neuron is found + _ = torch.neuron.device_count() + return torch.neuron.is_available() + except RuntimeError: + return False + + return hasattr(torch, "neuron") and torch.neuron.is_available() + + +def is_dvclive_available(): + return _is_package_available("dvclive") + + +def is_torchdata_available(): + return _is_package_available("torchdata") + + +# TODO: Remove this function once stateful_dataloader is a stable feature in torchdata. +def is_torchdata_stateful_dataloader_available(): + package_exists = _is_package_available("torchdata") + if package_exists: + torchdata_version = version.parse(importlib.metadata.version("torchdata")) + return compare_versions(torchdata_version, ">=", "0.8.0") + return False + + +def torchao_required(func): + """ + A decorator that ensures the decorated function is only called when torchao is available. + """ + + @wraps(func) + def wrapper(*args, **kwargs): + if not is_torchao_available(): + raise ImportError( + "`torchao` is not available, please install it before calling this function via `pip install torchao`." + ) + return func(*args, **kwargs) + + return wrapper + + +# TODO: Rework this into `utils.deepspeed` and migrate the "core" chunks into `accelerate.deepspeed` +def deepspeed_required(func): + """ + A decorator that ensures the decorated function is only called when deepspeed is enabled. + """ + + @wraps(func) + def wrapper(*args, **kwargs): + from accelerate.state import AcceleratorState + from accelerate.utils.dataclasses import DistributedType + + if AcceleratorState._shared_state != {} and AcceleratorState().distributed_type != DistributedType.DEEPSPEED: + raise ValueError( + "DeepSpeed is not enabled, please make sure that an `Accelerator` is configured for `deepspeed` " + "before calling this function." + ) + return func(*args, **kwargs) + + return wrapper + + +def is_weights_only_available(): + # Weights only with allowlist was added in 2.4.0 + # ref: https://github.com/pytorch/pytorch/pull/124331 + return is_torch_version(">=", "2.4.0") + + +def is_numpy_available(min_version="1.25.0"): + numpy_version = parse(importlib.metadata.version("numpy")) + return compare_versions(numpy_version, ">=", min_version) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/launch.py b/kbidu_env/Lib/site-packages/accelerate/utils/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..a1801c8c4335a445026b64bc00ff3fc4abe0414f --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/launch.py @@ -0,0 +1,827 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import subprocess +import sys +import warnings +from ast import literal_eval +from shutil import which +from typing import Any + +import torch + +from ..commands.config.config_args import SageMakerConfig +from ..utils import ( + DynamoBackend, + PrecisionType, + is_fp8_available, + is_hpu_available, + is_mlu_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_torch_xla_available, + is_xpu_available, +) +from ..utils.constants import DEEPSPEED_MULTINODE_LAUNCHERS +from ..utils.other import get_free_port, is_port_in_use, merge_dicts +from ..utils.versions import compare_versions +from . import parse_flag_from_env +from .dataclasses import DistributedType, SageMakerDistributedType + + +def _filter_args(args, parser, default_args=[]): + """ + Filters out all `accelerate` specific args + """ + new_args, _ = parser.parse_known_args(default_args) + for key, value in vars(args).items(): + if key in vars(new_args).keys(): + setattr(new_args, key, value) + return new_args + + +def _get_mpirun_args(): + """ + Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs + are: OpenMPI, Intel MPI, or MVAPICH. + + Returns: Program name and arg names for hostfile, num processes, and processes per node + """ + # Find the MPI program name + mpi_apps = [x for x in ["mpirun", "mpiexec"] if which(x)] + + if len(mpi_apps) == 0: + raise OSError("mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.") + + # Call the app with the --version flag to determine which MPI app is installed + mpi_app = mpi_apps[0] + mpirun_version = subprocess.check_output([mpi_app, "--version"]) + + if b"Open MPI" in mpirun_version: + return mpi_app, "--hostfile", "-n", "--npernode", "--bind-to" + else: + # Intel MPI and MVAPICH both use the same arg names + return mpi_app, "-f", "-n", "-ppn", "" + + +def setup_fp8_env(args: argparse.Namespace, current_env: dict[str, str]): + """ + Setup the FP8 environment variables. + """ + prefix = "ACCELERATE_" + for arg in vars(args): + if arg.startswith("fp8_"): + value = getattr(args, arg) + if value is not None: + if arg == "fp8_override_linear_precision": + current_env[prefix + "FP8_OVERRIDE_FPROP"] = str(value[0]) + current_env[prefix + "FP8_OVERRIDE_DGRAD"] = str(value[1]) + current_env[prefix + "FP8_OVERRIDE_WGRAD"] = str(value[2]) + else: + current_env[f"{prefix}{arg.upper()}"] = str(getattr(args, arg)) + return current_env + + +def prepare_simple_launcher_cmd_env(args: argparse.Namespace) -> tuple[list[str], dict[str, str]]: + """ + Prepares and returns the command list and an environment with the correct simple launcher environment variables. + """ + cmd = [] + if args.no_python and args.module: + raise ValueError("--module and --no_python cannot be used together") + + num_processes = getattr(args, "num_processes", None) + num_machines = args.num_machines + if args.mpirun_hostfile is not None: + mpi_app_name, hostfile_arg, num_proc_arg, proc_per_node_arg, bind_to_arg = _get_mpirun_args() + bind_to = getattr(args, "bind-to", "socket") + nproc_per_node = str(num_processes // num_machines) if num_processes and num_machines else "1" + cmd += [ + mpi_app_name, + hostfile_arg, + args.mpirun_hostfile, + proc_per_node_arg, + nproc_per_node, + ] + if num_processes: + cmd += [num_proc_arg, str(num_processes)] + if bind_to_arg: + cmd += [bind_to_arg, bind_to] + if not args.no_python: + cmd.append(sys.executable) + if args.module: + cmd.append("-m") + cmd.append(args.training_script) + cmd.extend(args.training_script_args) + + current_env = os.environ.copy() + current_env["ACCELERATE_USE_CPU"] = str(args.cpu or args.use_cpu) + if args.debug: + current_env["ACCELERATE_DEBUG_MODE"] = "true" + if args.gpu_ids != "all" and args.gpu_ids is not None: + if is_xpu_available(): + current_env["ZE_AFFINITY_MASK"] = args.gpu_ids + elif is_mlu_available(): + current_env["MLU_VISIBLE_DEVICES"] = args.gpu_ids + elif is_sdaa_available(): + current_env["SDAA_VISIBLE_DEVICES"] = args.gpu_ids + elif is_musa_available(): + current_env["MUSA_VISIBLE_DEVICES"] = args.gpu_ids + elif is_npu_available(): + current_env["ASCEND_RT_VISIBLE_DEVICES"] = args.gpu_ids + elif is_hpu_available(): + current_env["HABANA_VISIBLE_MODULES"] = args.gpu_ids + elif is_neuron_available(): + current_env["NEURON_RT_VISIBLE_CORES"] = args.gpu_ids + else: + current_env["CUDA_VISIBLE_DEVICES"] = args.gpu_ids + if num_machines > 1: + assert args.main_process_ip is not None, ( + "When using multiple machines, you need to specify the main process IP." + ) + assert args.main_process_port is not None, ( + "When using multiple machines, you need to specify the main process port." + ) + + if (num_processes is not None and num_processes > 1) or num_machines > 1: + current_env["MASTER_ADDR"] = args.main_process_ip if args.main_process_ip is not None else "127.0.0.1" + current_env["MASTER_PORT"] = str(args.main_process_port) if args.main_process_port is not None else "29500" + if parse_flag_from_env(current_env["ACCELERATE_USE_CPU"], False): + current_env["KMP_AFFINITY"] = "granularity=fine,compact,1,0" + current_env["KMP_BLOCKTIME"] = str(1) + + try: + mixed_precision = PrecisionType(args.mixed_precision.lower()) + except ValueError: + raise ValueError( + f"Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}." + ) + + current_env["ACCELERATE_MIXED_PRECISION"] = str(mixed_precision) + if args.mixed_precision.lower() == "fp8": + if not is_fp8_available(): + raise RuntimeError( + "FP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed." + ) + current_env = setup_fp8_env(args, current_env) + + try: + dynamo_backend = DynamoBackend(args.dynamo_backend.upper()) + except ValueError: + raise ValueError( + f"Unknown dynamo backend: {args.dynamo_backend.upper()}. Choose between {DynamoBackend.list()}." + ) + current_env["ACCELERATE_DYNAMO_BACKEND"] = dynamo_backend.value + current_env["ACCELERATE_DYNAMO_MODE"] = args.dynamo_mode + current_env["ACCELERATE_DYNAMO_USE_FULLGRAPH"] = str(args.dynamo_use_fullgraph) + current_env["ACCELERATE_DYNAMO_USE_DYNAMIC"] = str(args.dynamo_use_dynamic) + current_env["ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATION"] = str(args.dynamo_use_regional_compilation) + + current_env["OMP_NUM_THREADS"] = str(args.num_cpu_threads_per_process) + if args.enable_cpu_affinity: + current_env["ACCELERATE_CPU_AFFINITY"] = "1" + return cmd, current_env + + +def prepare_multi_gpu_env(args: argparse.Namespace) -> dict[str, str]: + """ + Prepares and returns an environment with the correct multi-GPU environment variables. + """ + # get free port and update configurations + if args.main_process_port == 0: + args.main_process_port = get_free_port() + + elif args.main_process_port is None: + args.main_process_port = 29500 + + num_processes = args.num_processes + num_machines = args.num_machines + main_process_ip = args.main_process_ip + main_process_port = args.main_process_port + if num_machines > 1: + args.nproc_per_node = str(num_processes // num_machines) + args.nnodes = str(num_machines) + args.node_rank = int(args.machine_rank) + if getattr(args, "same_network", False): + args.master_addr = str(main_process_ip) + args.master_port = str(main_process_port) + else: + args.rdzv_endpoint = f"{main_process_ip}:{main_process_port}" + else: + args.nproc_per_node = str(num_processes) + if main_process_port is not None: + args.master_port = str(main_process_port) + + # only need to check port availability in main process, in case we have to start multiple launchers on the same machine + # for some reasons like splitting log files. + need_port_check = num_machines <= 1 or int(args.machine_rank) == 0 + if need_port_check and is_port_in_use(main_process_port): + if num_machines <= 1: + args.standalone = True + warnings.warn( + f"Port `{main_process_port}` is already in use. " + "Accelerate will attempt to launch in a standalone-like mode by finding an open port automatically for this session. " + "If this current attempt fails, or for more control in future runs, please specify a different port " + "(e.g., `--main_process_port `) or use `--main_process_port 0` for automatic selection " + "in your launch command or Accelerate config file." + ) + else: + raise ConnectionError( + f"Tried to launch distributed communication on port `{main_process_port}`, but another process is utilizing it. " + "Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file)" + " and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`." + ) + + if args.module and args.no_python: + raise ValueError("--module and --no_python cannot be used together") + elif args.module: + args.module = True + elif args.no_python: + args.no_python = True + + current_env = os.environ.copy() + if args.debug: + current_env["ACCELERATE_DEBUG_MODE"] = "true" + gpu_ids = getattr(args, "gpu_ids", "all") + if gpu_ids != "all" and args.gpu_ids is not None: + if is_xpu_available(): + current_env["ZE_AFFINITY_MASK"] = gpu_ids + elif is_mlu_available(): + current_env["MLU_VISIBLE_DEVICES"] = gpu_ids + elif is_sdaa_available(): + current_env["SDAA_VISIBLE_DEVICES"] = gpu_ids + elif is_musa_available(): + current_env["MUSA_VISIBLE_DEVICES"] = gpu_ids + elif is_npu_available(): + current_env["ASCEND_RT_VISIBLE_DEVICES"] = gpu_ids + elif is_hpu_available(): + current_env["HABANA_VISIBLE_MODULES"] = gpu_ids + elif is_neuron_available(): + current_env["NEURON_RT_VISIBLE_CORES"] = gpu_ids + else: + current_env["CUDA_VISIBLE_DEVICES"] = gpu_ids + mixed_precision = args.mixed_precision.lower() + try: + mixed_precision = PrecisionType(mixed_precision) + except ValueError: + raise ValueError(f"Unknown mixed_precision mode: {mixed_precision}. Choose between {PrecisionType.list()}.") + + current_env["ACCELERATE_MIXED_PRECISION"] = str(mixed_precision) + if args.mixed_precision.lower() == "fp8": + if not is_fp8_available(): + raise RuntimeError( + "FP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed." + ) + current_env = setup_fp8_env(args, current_env) + + try: + dynamo_backend = DynamoBackend(args.dynamo_backend.upper()) + except ValueError: + raise ValueError( + f"Unknown dynamo backend: {args.dynamo_backend.upper()}. Choose between {DynamoBackend.list()}." + ) + current_env["ACCELERATE_DYNAMO_BACKEND"] = dynamo_backend.value + current_env["ACCELERATE_DYNAMO_MODE"] = args.dynamo_mode + current_env["ACCELERATE_DYNAMO_USE_FULLGRAPH"] = str(args.dynamo_use_fullgraph) + current_env["ACCELERATE_DYNAMO_USE_DYNAMIC"] = str(args.dynamo_use_dynamic) + current_env["ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATION"] = str(args.dynamo_use_regional_compilation) + + if args.use_fsdp: + current_env["ACCELERATE_USE_FSDP"] = "true" + if args.fsdp_cpu_ram_efficient_loading and not args.fsdp_sync_module_states: + raise ValueError("When using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`") + + current_env["FSDP_VERSION"] = str(args.fsdp_version) if hasattr(args, "fsdp_version") else "1" + + # For backwards compatibility, we support this in launched scripts, + # however, we do not ask users for this in `accelerate config` CLI + current_env["FSDP_SHARDING_STRATEGY"] = str(args.fsdp_sharding_strategy) + + current_env["FSDP_RESHARD_AFTER_FORWARD"] = str(args.fsdp_reshard_after_forward).lower() + current_env["FSDP_OFFLOAD_PARAMS"] = str(args.fsdp_offload_params).lower() + current_env["FSDP_MIN_NUM_PARAMS"] = str(args.fsdp_min_num_params) + if args.fsdp_auto_wrap_policy is not None: + current_env["FSDP_AUTO_WRAP_POLICY"] = str(args.fsdp_auto_wrap_policy) + if args.fsdp_transformer_layer_cls_to_wrap is not None: + current_env["FSDP_TRANSFORMER_CLS_TO_WRAP"] = str(args.fsdp_transformer_layer_cls_to_wrap) + if args.fsdp_backward_prefetch is not None: + current_env["FSDP_BACKWARD_PREFETCH"] = str(args.fsdp_backward_prefetch) + if args.fsdp_state_dict_type is not None: + current_env["FSDP_STATE_DICT_TYPE"] = str(args.fsdp_state_dict_type) + current_env["FSDP_FORWARD_PREFETCH"] = str(args.fsdp_forward_prefetch).lower() + current_env["FSDP_USE_ORIG_PARAMS"] = str(args.fsdp_use_orig_params).lower() + current_env["FSDP_CPU_RAM_EFFICIENT_LOADING"] = str(args.fsdp_cpu_ram_efficient_loading).lower() + current_env["FSDP_SYNC_MODULE_STATES"] = str(args.fsdp_sync_module_states).lower() + current_env["FSDP_ACTIVATION_CHECKPOINTING"] = str(args.fsdp_activation_checkpointing).lower() + if getattr(args, "fsdp_ignored_modules", None) is not None: + current_env["FSDP_IGNORED_MODULES"] = str(args.fsdp_ignored_modules) + + if args.use_megatron_lm: + prefix = "MEGATRON_LM_" + current_env["ACCELERATE_USE_MEGATRON_LM"] = "true" + current_env[prefix + "TP_DEGREE"] = str(args.megatron_lm_tp_degree) + current_env[prefix + "USE_CUSTOM_FSDP"] = str(args.megatron_lm_use_custom_fsdp) + if args.megatron_lm_no_load_optim is not None: + current_env[prefix + "NO_LOAD_OPTIM"] = str(args.megatron_lm_no_load_optim) + if args.megatron_lm_eod_mask_loss is not None: + current_env[prefix + "EOD_MASK_LOSS"] = str(args.megatron_lm_eod_mask_loss) + if args.megatron_lm_no_save_optim is not None: + current_env[prefix + "NO_SAVE_OPTIM"] = str(args.megatron_lm_no_save_optim) + if args.megatron_lm_optimizer_cpu_offload is not None: + current_env[prefix + "OPTIMIZER_CPU_OFFLOAD"] = str(args.megatron_lm_optimizer_cpu_offload) + if args.megatron_lm_use_precision_aware_optimizer is not None: + current_env[prefix + "USE_PRECISION_AWARE_OPTIMIZER"] = str(args.megatron_lm_use_precision_aware_optimizer) + if args.megatron_lm_overlap_cpu_optimizer_d2h_h2d is not None: + current_env[prefix + "OVERLAP_CPU_OPTIMIZER_D2H_H2D"] = str(args.megatron_lm_overlap_cpu_optimizer_d2h_h2d) + if args.megatron_lm_decoder_last_pipeline_num_layers is not None: + current_env[prefix + "DECODER_LAST_PIPELINE_NUM_LAYERS"] = str( + args.megatron_lm_decoder_last_pipeline_num_layers + ) + current_env[prefix + "PP_DEGREE"] = str(args.megatron_lm_pp_degree) + current_env[prefix + "GRADIENT_CLIPPING"] = str(args.megatron_lm_gradient_clipping) + if args.megatron_lm_num_micro_batches is not None: + current_env[prefix + "NUM_MICRO_BATCHES"] = str(args.megatron_lm_num_micro_batches) + if args.megatron_lm_sequence_parallelism is not None: + current_env[prefix + "SEQUENCE_PARALLELISM"] = str(args.megatron_lm_sequence_parallelism) + if args.megatron_lm_recompute_activations is not None: + current_env[prefix + "RECOMPUTE_ACTIVATIONS"] = str(args.megatron_lm_recompute_activations) + if args.megatron_lm_use_distributed_optimizer is not None: + current_env[prefix + "USE_DISTRIBUTED_OPTIMIZER"] = str(args.megatron_lm_use_distributed_optimizer) + if args.megatron_lm_recompute_granularity is not None: + current_env[prefix + "RECOMPUTE_GRANULARITY"] = str(args.megatron_lm_recompute_granularity) + if args.megatron_lm_recompute_method is not None: + current_env[prefix + "RECOMPUTE_METHOD"] = str(args.megatron_lm_recompute_method) + if args.megatron_lm_recompute_num_layers is not None: + current_env[prefix + "RECOMPUTE_NUM_LAYERS"] = str(args.megatron_lm_recompute_num_layers) + if args.megatron_lm_attention_backend is not None: + current_env[prefix + "ATTENTION_BACKEND"] = str(args.megatron_lm_attention_backend) + if args.megatron_lm_expert_model_parallel_size is not None: + current_env[prefix + "EXPERT_MODEL_PARALLEL_SIZE"] = str(args.megatron_lm_expert_model_parallel_size) + if args.megatron_lm_context_parallel_size is not None: + current_env[prefix + "CONTEXT_PARALLEL_SIZE"] = str(args.megatron_lm_context_parallel_size) + if args.megatron_lm_attention_dropout is not None: + current_env[prefix + "ATTENTION_DROPOUT"] = str(args.megatron_lm_attention_dropout) + if args.megatron_lm_hidden_dropout is not None: + current_env[prefix + "HIDDEN_DROPOUT"] = str(args.megatron_lm_hidden_dropout) + if args.megatron_lm_attention_softmax_in_fp32 is not None: + current_env[prefix + "ATTENTION_SOFTMAX_IN_FP32"] = str(args.megatron_lm_attention_softmax_in_fp32) + if args.megatron_lm_expert_tensor_parallel_size is not None: + current_env[prefix + "EXPERT_TENSOR_PARALLEL_SIZE"] = str(args.megatron_lm_expert_tensor_parallel_size) + if args.megatron_lm_calculate_per_token_loss is not None: + current_env[prefix + "CALCULATE_PER_TOKEN_LOSS"] = str(args.megatron_lm_calculate_per_token_loss) + if args.megatron_lm_use_rotary_position_embeddings is not None: + current_env[prefix + "USE_ROTARY_POSITION_EMBEDDINGS"] = str( + args.megatron_lm_use_rotary_position_embeddings + ) + + current_env["OMP_NUM_THREADS"] = str(args.num_cpu_threads_per_process) + if args.enable_cpu_affinity: + current_env["ACCELERATE_CPU_AFFINITY"] = "1" + + if args.use_parallelism_config: + current_env = prepare_extend_env_parallelism_config(args, current_env) + + return current_env + + +def prepare_extend_env_parallelism_config( + args: argparse.Namespace, current_env: dict +) -> tuple[list[str], dict[str, str]]: + """ + Extends `current_env` with context parallelism env vars if any have been set + """ + + prefix = "PARALLELISM_CONFIG_" + + current_env["ACCELERATE_USE_PARALLELISM_CONFIG"] = "true" + current_env[prefix + "DP_REPLICATE_SIZE"] = str(args.parallelism_config_dp_replicate_size) + current_env[prefix + "DP_SHARD_SIZE"] = str(args.parallelism_config_dp_shard_size) + current_env[prefix + "TP_SIZE"] = str(args.parallelism_config_tp_size) + current_env[prefix + "CP_SIZE"] = str(args.parallelism_config_cp_size) + current_env[prefix + "CP_BACKEND"] = str(args.parallelism_config_cp_backend) + current_env[prefix + "SP_SIZE"] = str(args.parallelism_config_sp_size) + current_env[prefix + "SP_BACKEND"] = str(args.parallelism_config_sp_backend) + if args.parallelism_config_cp_size > 1: + current_env[prefix + "CP_COMM_STRATEGY"] = str(args.parallelism_config_cp_comm_strategy) + if args.parallelism_config_sp_size > 1: + current_env[prefix + "SP_SEQ_LENGTH"] = str(args.parallelism_config_sp_seq_length) + current_env[prefix + "SP_SEQ_LENGTH_IS_VARIABLE"] = str(args.parallelism_config_sp_seq_length_is_variable) + current_env[prefix + "SP_ATTN_IMPLEMENTATION"] = str(args.parallelism_config_sp_attn_implementation) + + return current_env + + +def prepare_deepspeed_cmd_env(args: argparse.Namespace) -> tuple[list[str], dict[str, str]]: + """ + Prepares and returns the command list and an environment with the correct DeepSpeed environment variables. + """ + # get free port and update configurations + if args.main_process_port == 0: + args.main_process_port = get_free_port() + + elif args.main_process_port is None: + args.main_process_port = 29500 + + num_processes = args.num_processes + num_machines = args.num_machines + main_process_ip = args.main_process_ip + main_process_port = args.main_process_port + cmd = None + + # make sure launcher is not None + if args.deepspeed_multinode_launcher is None: + # set to default pdsh + args.deepspeed_multinode_launcher = DEEPSPEED_MULTINODE_LAUNCHERS[0] + + if num_machines > 1 and args.deepspeed_multinode_launcher != DEEPSPEED_MULTINODE_LAUNCHERS[1]: + cmd = ["deepspeed"] + cmd.extend(["--hostfile", str(args.deepspeed_hostfile)]) + if args.deepspeed_multinode_launcher == "nossh": + if compare_versions("deepspeed", "<", "0.14.5"): + raise ValueError("nossh launcher requires DeepSpeed >= 0.14.5") + cmd.extend(["--node_rank", str(args.machine_rank), "--no_ssh"]) + else: + cmd.extend(["--no_local_rank", "--launcher", str(args.deepspeed_multinode_launcher)]) + if args.deepspeed_exclusion_filter is not None: + cmd.extend( + [ + "--exclude", + str(args.deepspeed_exclusion_filter), + ] + ) + elif args.deepspeed_inclusion_filter is not None: + cmd.extend( + [ + "--include", + str(args.deepspeed_inclusion_filter), + ] + ) + else: + cmd.extend(["--num_gpus", str(args.num_processes // args.num_machines)]) + if main_process_ip: + cmd.extend(["--master_addr", str(main_process_ip)]) + cmd.extend(["--master_port", str(main_process_port)]) + if args.module and args.no_python: + raise ValueError("--module and --no_python cannot be used together") + elif args.module: + cmd.append("--module") + elif args.no_python: + cmd.append("--no_python") + cmd.append(args.training_script) + cmd.extend(args.training_script_args) + elif num_machines > 1 and args.deepspeed_multinode_launcher == DEEPSPEED_MULTINODE_LAUNCHERS[1]: + args.nproc_per_node = str(num_processes // num_machines) + args.nnodes = str(num_machines) + args.node_rank = int(args.machine_rank) + if getattr(args, "same_network", False): + args.master_addr = str(main_process_ip) + args.master_port = str(main_process_port) + else: + args.rdzv_endpoint = f"{main_process_ip}:{main_process_port}" + else: + args.nproc_per_node = str(num_processes) + if main_process_port is not None: + args.master_port = str(main_process_port) + + # only need to check port availability in main process, in case we have to start multiple launchers on the same machine + # for some reasons like splitting log files. + need_port_check = num_machines <= 1 or int(args.machine_rank) == 0 + if need_port_check and is_port_in_use(main_process_port): + if num_machines <= 1: + args.standalone = True + warnings.warn( + f"Port `{main_process_port}` is already in use. " + "Accelerate will attempt to launch in a standalone-like mode by finding an open port automatically for this session. " + "If this current attempt fails, or for more control in future runs, please specify a different port " + "(e.g., `--main_process_port `) or use `--main_process_port 0` for automatic selection " + "in your launch command or Accelerate config file." + ) + else: + raise ConnectionError( + f"Tried to launch distributed communication on port `{main_process_port}`, but another process is utilizing it. " + "Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file)" + " and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`." + ) + + if args.module and args.no_python: + raise ValueError("--module and --no_python cannot be used together") + elif args.module: + args.module = True + elif args.no_python: + args.no_python = True + + current_env = os.environ.copy() + if args.debug: + current_env["ACCELERATE_DEBUG_MODE"] = "true" + gpu_ids = getattr(args, "gpu_ids", "all") + if gpu_ids != "all" and args.gpu_ids is not None: + if is_xpu_available(): + current_env["ZE_AFFINITY_MASK"] = gpu_ids + elif is_mlu_available(): + current_env["MLU_VISIBLE_DEVICES"] = gpu_ids + elif is_sdaa_available(): + current_env["SDAA_VISIBLE_DEVICES"] = gpu_ids + elif is_musa_available(): + current_env["MUSA_VISIBLE_DEVICES"] = gpu_ids + elif is_npu_available(): + current_env["ASCEND_RT_VISIBLE_DEVICES"] = gpu_ids + elif is_hpu_available(): + current_env["HABANA_VISIBLE_MODULES"] = gpu_ids + elif is_neuron_available(): + current_env["NEURON_RT_VISIBLE_CORES"] = gpu_ids + else: + current_env["CUDA_VISIBLE_DEVICES"] = gpu_ids + try: + mixed_precision = PrecisionType(args.mixed_precision.lower()) + except ValueError: + raise ValueError( + f"Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}." + ) + + current_env["PYTHONPATH"] = env_var_path_add("PYTHONPATH", os.path.abspath(".")) + current_env["ACCELERATE_MIXED_PRECISION"] = str(mixed_precision) + if args.mixed_precision.lower() == "fp8": + if not is_fp8_available(): + raise RuntimeError( + "FP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed." + ) + current_env = setup_fp8_env(args, current_env) + current_env["ACCELERATE_CONFIG_DS_FIELDS"] = str(args.deepspeed_fields_from_accelerate_config).lower() + current_env["ACCELERATE_USE_DEEPSPEED"] = "true" + if args.zero_stage is not None: + current_env["ACCELERATE_DEEPSPEED_ZERO_STAGE"] = str(args.zero_stage) + if args.gradient_accumulation_steps is not None: + current_env["ACCELERATE_GRADIENT_ACCUMULATION_STEPS"] = str(args.gradient_accumulation_steps) + if args.gradient_clipping is not None: + current_env["ACCELERATE_GRADIENT_CLIPPING"] = str(args.gradient_clipping).lower() + if args.offload_optimizer_device is not None: + current_env["ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE"] = str(args.offload_optimizer_device).lower() + if args.offload_param_device is not None: + current_env["ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICE"] = str(args.offload_param_device).lower() + if args.zero3_init_flag is not None: + current_env["ACCELERATE_DEEPSPEED_ZERO3_INIT"] = str(args.zero3_init_flag).lower() + if args.zero3_save_16bit_model is not None: + current_env["ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL"] = str(args.zero3_save_16bit_model).lower() + if args.deepspeed_config_file is not None: + current_env["ACCELERATE_DEEPSPEED_CONFIG_FILE"] = str(args.deepspeed_config_file) + if args.enable_cpu_affinity: + current_env["ACCELERATE_CPU_AFFINITY"] = "1" + if args.deepspeed_moe_layer_cls_names is not None: + current_env["ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES"] = str(args.deepspeed_moe_layer_cls_names) + + if args.use_parallelism_config: + current_env = prepare_extend_env_parallelism_config(args, current_env) + + return cmd, current_env + + +def prepare_tpu( + args: argparse.Namespace, current_env: dict[str, str], pod: bool = False +) -> tuple[argparse.Namespace, dict[str, str]]: + """ + Prepares and returns an environment with the correct TPU environment variables. + """ + if args.mixed_precision == "bf16" and is_torch_xla_available(check_is_tpu=True): + if args.downcast_bf16: + current_env["XLA_DOWNCAST_BF16"] = "1" + else: + current_env["XLA_USE_BF16"] = "1" + if args.debug: + current_env["ACCELERATE_DEBUG_MODE"] = "true" + if pod: + # Take explicit args and set them up for XLA + args.vm = args.tpu_vm + args.tpu = args.tpu_name + return args, current_env + + +def _convert_nargs_to_dict(nargs: list[str]) -> dict[str, str]: + if len(nargs) < 0: + return {} + # helper function to infer type for argsparser + + def _infer_type(s): + try: + s = float(s) + + if s // 1 == s: + return int(s) + return s + except ValueError: + return s + + parser = argparse.ArgumentParser() + _, unknown = parser.parse_known_args(nargs) + for index, argument in enumerate(unknown): + if argument.startswith(("-", "--")): + action = None + if index + 1 < len(unknown): # checks if next index would be in list + if unknown[index + 1].startswith(("-", "--")): # checks if next element is an key + # raise an error if element is store_true or store_false + raise ValueError( + "SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types" + ) + else: # raise an error if last element is store_true or store_false + raise ValueError( + "SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types" + ) + # adds argument to parser based on action_store true + if action is None: + parser.add_argument(argument, type=_infer_type) + else: + parser.add_argument(argument, action=action) + + return { + key: (literal_eval(value) if value in ("True", "False") else value) + for key, value in parser.parse_args(nargs).__dict__.items() + } + + +def prepare_sagemager_args_inputs( + sagemaker_config: SageMakerConfig, args: argparse.Namespace +) -> tuple[argparse.Namespace, dict[str, Any]]: + # configure environment + print("Configuring Amazon SageMaker environment") + os.environ["AWS_DEFAULT_REGION"] = sagemaker_config.region + + # configure credentials + if sagemaker_config.profile is not None: + os.environ["AWS_PROFILE"] = sagemaker_config.profile + elif args.aws_access_key_id is not None and args.aws_secret_access_key is not None: + os.environ["AWS_ACCESS_KEY_ID"] = args.aws_access_key_id + os.environ["AWS_SECRET_ACCESS_KEY"] = args.aws_secret_access_key + else: + raise OSError("You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profile") + + # extract needed arguments + source_dir = os.path.dirname(args.training_script) + if not source_dir: # checks if string is empty + source_dir = "." + entry_point = os.path.basename(args.training_script) + if not entry_point.endswith(".py"): + raise ValueError(f'Your training script should be a python script and not "{entry_point}"') + + print("Converting Arguments to Hyperparameters") + hyperparameters = _convert_nargs_to_dict(args.training_script_args) + + try: + mixed_precision = PrecisionType(args.mixed_precision.lower()) + except ValueError: + raise ValueError( + f"Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}." + ) + + try: + dynamo_backend = DynamoBackend(args.dynamo_backend.upper()) + except ValueError: + raise ValueError( + f"Unknown dynamo backend: {args.dynamo_backend.upper()}. Choose between {DynamoBackend.list()}." + ) + + # Environment variables to be set for use during training job + environment = { + "ACCELERATE_USE_SAGEMAKER": "true", + "ACCELERATE_MIXED_PRECISION": str(mixed_precision), + "ACCELERATE_DYNAMO_BACKEND": dynamo_backend.value, + "ACCELERATE_DYNAMO_MODE": args.dynamo_mode, + "ACCELERATE_DYNAMO_USE_FULLGRAPH": str(args.dynamo_use_fullgraph), + "ACCELERATE_DYNAMO_USE_DYNAMIC": str(args.dynamo_use_dynamic), + "ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATION": str(args.dynamo_use_regional_compilation), + "ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPE": sagemaker_config.distributed_type.value, + } + if args.mixed_precision.lower() == "fp8": + if not is_fp8_available(): + raise RuntimeError( + "FP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed." + ) + environment = setup_fp8_env(args, environment) + # configure distribution set up + distribution = None + if sagemaker_config.distributed_type == SageMakerDistributedType.DATA_PARALLEL: + distribution = {"smdistributed": {"dataparallel": {"enabled": True}}} + + # configure sagemaker inputs + sagemaker_inputs = None + if sagemaker_config.sagemaker_inputs_file is not None: + print(f"Loading SageMaker Inputs from {sagemaker_config.sagemaker_inputs_file} file") + sagemaker_inputs = {} + with open(sagemaker_config.sagemaker_inputs_file) as file: + for i, line in enumerate(file): + if i == 0: + continue + l = line.split("\t") + sagemaker_inputs[l[0]] = l[1].strip() + print(f"Loaded SageMaker Inputs: {sagemaker_inputs}") + + # configure sagemaker metrics + sagemaker_metrics = None + if sagemaker_config.sagemaker_metrics_file is not None: + print(f"Loading SageMaker Metrics from {sagemaker_config.sagemaker_metrics_file} file") + sagemaker_metrics = [] + with open(sagemaker_config.sagemaker_metrics_file) as file: + for i, line in enumerate(file): + if i == 0: + continue + l = line.split("\t") + metric_dict = { + "Name": l[0], + "Regex": l[1].strip(), + } + sagemaker_metrics.append(metric_dict) + print(f"Loaded SageMaker Metrics: {sagemaker_metrics}") + + # configure session + print("Creating Estimator") + args = { + "image_uri": sagemaker_config.image_uri, + "entry_point": entry_point, + "source_dir": source_dir, + "role": sagemaker_config.iam_role_name, + "transformers_version": sagemaker_config.transformers_version, + "pytorch_version": sagemaker_config.pytorch_version, + "py_version": sagemaker_config.py_version, + "base_job_name": sagemaker_config.base_job_name, + "instance_count": sagemaker_config.num_machines, + "instance_type": sagemaker_config.ec2_instance_type, + "debugger_hook_config": False, + "distribution": distribution, + "hyperparameters": hyperparameters, + "environment": environment, + "metric_definitions": sagemaker_metrics, + } + + if sagemaker_config.additional_args is not None: + args = merge_dicts(sagemaker_config.additional_args, args) + return args, sagemaker_inputs + + +def env_var_path_add(env_var_name, path_to_add): + """ + Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the + caller to set it in os.environ. + """ + paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0] + paths.append(str(path_to_add)) + return ":".join(paths) + + +class PrepareForLaunch: + """ + Prepare a function that will launched in a distributed setup. + + Args: + launcher (`Callable`): + The function to launch. + distributed_type ([`~state.DistributedType`]): + The distributed type to prepare for. + debug (`bool`, *optional*, defaults to `False`): + Whether or not this is a debug launch. + """ + + def __init__(self, launcher, distributed_type="NO", debug=False): + self.launcher = launcher + self.distributed_type = DistributedType(distributed_type) + self.debug = debug + + def __call__(self, index, *args): + if self.debug: + world_size = int(os.environ.get("WORLD_SIZE")) + rdv_file = os.environ.get("ACCELERATE_DEBUG_RDV_FILE") + torch.distributed.init_process_group( + "gloo", + rank=index, + store=torch.distributed.FileStore(rdv_file, world_size), + world_size=world_size, + ) + elif self.distributed_type in ( + DistributedType.MULTI_GPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_CPU, + DistributedType.MULTI_NEURON, + ): + # Prepare the environment for torch.distributed + os.environ["LOCAL_RANK"] = str(index) + nproc = int(os.environ.get("NPROC", 1)) + node_rank = int(os.environ.get("NODE_RANK", 0)) + os.environ["RANK"] = str(nproc * node_rank + index) + + os.environ["FORK_LAUNCHED"] = str(1) + self.launcher(*args) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/megatron_lm.py b/kbidu_env/Lib/site-packages/accelerate/utils/megatron_lm.py new file mode 100644 index 0000000000000000000000000000000000000000..ff8900cbad888af9dceca2adc2600a02d213d304 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/megatron_lm.py @@ -0,0 +1,1248 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import math +import os +from abc import ABC +from functools import partial + +import torch +import torch.nn.functional as F +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from ..optimizer import AcceleratedOptimizer +from ..scheduler import AcceleratedScheduler +from .imports import is_megatron_lm_available +from .operations import recursively_apply, send_to_device + + +if is_megatron_lm_available(): + from megatron.core import mpu, tensor_parallel + from megatron.core.distributed import DistributedDataParallel as LocalDDP + from megatron.core.distributed import finalize_model_grads + from megatron.core.enums import ModelType + from megatron.core.num_microbatches_calculator import get_num_microbatches + from megatron.core.optimizer import get_megatron_optimizer + from megatron.core.parallel_state import get_tensor_model_parallel_group, get_tensor_model_parallel_src_rank + from megatron.core.pipeline_parallel import get_forward_backward_func + from megatron.core.utils import get_model_config + from megatron.legacy.data.dataset_utils import build_train_valid_test_datasets + from megatron.legacy.model import BertModel, T5Model + from megatron.legacy.model.classification import Classification + from megatron.training import ( + get_args, + get_tensorboard_writer, + get_tokenizer, + print_rank_last, + ) + from megatron.training.arguments import ( + _add_data_args, + _add_validation_args, + core_transformer_config_from_args, + parse_args, + validate_args, + ) + from megatron.training.checkpointing import load_args_from_checkpoint, load_checkpoint, save_checkpoint + from megatron.training.global_vars import set_global_variables + from megatron.training.gpt_builders import gpt_builder + from megatron.training.initialize import ( + _compile_dependencies, + _init_autoresume, + _initialize_distributed, + _set_random_seed, + set_jit_fusion_options, + write_args_to_tensorboard, + ) + from megatron.training.tokenizer.tokenizer import _vocab_size_with_padding + from megatron.training.training import ( + build_train_valid_test_data_iterators, + get_optimizer_param_scheduler, + num_floating_point_operations, + setup_model_and_optimizer, + train_step, + training_log, + ) + from megatron.training.utils import ( + average_losses_across_data_parallel_group, + calc_params_l2_norm, + get_ltor_masks_and_position_ids, + ) + + +# model utilities +def model_provider_func(pre_process=True, post_process=True, add_encoder=True, add_decoder=True): + """Build the model.""" + args = get_args() + mode = "pre-training" if args.pretraining_flag else "fine-tuning" + if args.rank == 0: + print(f"Building {args.model_type_name} model in the {mode} mode.") + print( + "The Megatron LM model weights are initialized at random in `accelerator.prepare`. " + "Please use `accelerator.load_checkpoint` to load a pre-trained checkpoint matching the distributed setup." + ) + config = core_transformer_config_from_args(args) + if args.model_type_name == "bert": + if args.pretraining_flag: + num_tokentypes = 2 if args.bert_binary_head else 0 + model = BertModel( + config=config, + num_tokentypes=num_tokentypes, + add_binary_head=args.bert_binary_head, + parallel_output=True, + pre_process=pre_process, + post_process=post_process, + ) + else: + model = Classification( + config=config, + num_classes=args.num_labels, + num_tokentypes=2, + pre_process=pre_process, + post_process=post_process, + ) + elif args.model_type_name == "gpt": + # use the latest gpt builder to build the model and set use_legacy_models to False + args.use_legacy_models = False + model = gpt_builder(args, pre_process, post_process, vp_stage=None, config=None) + elif args.model_type_name == "t5": + model = T5Model( + config=config, + num_tokentypes=0, + parallel_output=True, + pre_process=pre_process, + post_process=post_process, + add_encoder=add_encoder, + add_decoder=add_decoder, + ) + else: + raise ValueError(f"Unsupported model type: {args.model_type_name}") + return model + + +def prepare_model_optimizer_scheduler(accelerator): + accelerator.print("Preparing model optimizer scheduler") + args = get_args() + if accelerator.state.megatron_lm_plugin.custom_prepare_model_function is not None: + if accelerator.state.megatron_lm_plugin.custom_model_provider_function is None: + raise ValueError( + "You must provide a `custom_model_provider_function` when using a `custom_prepare_model_function`." + ) + custom_model_provider_func = accelerator.state.megatron_lm_plugin.custom_model_provider_function + model = accelerator.state.megatron_lm_plugin.custom_prepare_model_function(custom_model_provider_func) + optimizer = prepare_optimizer(accelerator, model) + scheduler = prepare_scheduler(accelerator, optimizer, scheduler=None) + else: + model_type = ModelType.encoder_or_decoder + if args.model_type_name == "t5": + model_type = ModelType.encoder_and_decoder + model_provider_func_ = model_provider_func + if accelerator.state.megatron_lm_plugin.custom_model_provider_function is not None: + model_provider_func_ = accelerator.state.megatron_lm_plugin.custom_model_provider_function + (model, optimizer, scheduler) = setup_model_and_optimizer( + model_provider_func_, + model_type, + ) + args.model_len = len(model) + return model, optimizer, scheduler + + +# dataloader utilities +class MegatronLMDummyDataLoader: + """ + Dummy dataloader presents model parameters or param groups, this is primarily used to follow conventional training + + Args: + **dataset_kwargs: Megatron data arguments. + """ + + def __init__(self, **dataset_kwargs): + parser = argparse.ArgumentParser() + parser = _add_data_args(parser) + parser = _add_validation_args(parser) + data_args = parser.parse_known_args() + self.dataset_args = vars(data_args[0]) + self.dataset_args.update(dataset_kwargs) + self.dataset_args["megatron_dataset_flag"] = True + + def set_megatron_data_args(self): + args = get_args() + for key, value in self.dataset_args.items(): + old_value = getattr(args, key, "") + if old_value != value: + print( + f"WARNING: MegatronLMDummyDataLoader overriding arguments for {key}:{old_value} with {key}:{value}" + ) + setattr(args, key, value) + + def get_train_valid_test_datasets_provider(self, accelerator): + def train_valid_test_datasets_provider(train_val_test_num_samples): + """Build train, valid, and test datasets.""" + args = get_args() + dataset_args = { + "data_prefix": args.data_path if isinstance(args.data_path, (list, tuple)) else [args.data_path], + "splits_string": args.split, + "train_valid_test_num_samples": train_val_test_num_samples, + "seed": args.seed, + } + if args.model_type_name == "bert": + dataset_args.update( + { + "max_seq_length": args.seq_length, + "binary_head": args.bert_binary_head, + } + ) + elif args.model_type_name == "gpt": + dataset_args.update( + { + "max_seq_length": args.seq_length, + } + ) + elif args.model_type_name == "t5": + dataset_args.update( + { + "max_seq_length": args.encoder_seq_length, + "max_seq_length_dec": args.decoder_seq_length, + "dataset_type": "t5", + } + ) + else: + raise ValueError(f"Unsupported model type: {args.model_type_name}") + train_ds, valid_ds, test_ds = build_train_valid_test_datasets(**dataset_args) + return train_ds, valid_ds, test_ds + + if accelerator.state.megatron_lm_plugin.custom_megatron_datasets_provider_function is not None: + return accelerator.state.megatron_lm_plugin.custom_megatron_datasets_provider_function + try: + args = get_args() + # Use '--no-use-pep517 -e' to pip install nvidia's megatron from source + if args.model_type_name == "bert": + from pretrain_bert import train_valid_test_datasets_provider + + train_valid_test_datasets_provider.is_distributed = True + return train_valid_test_datasets_provider + elif args.model_type_name == "gpt": + from pretrain_gpt import train_valid_test_datasets_provider + + train_valid_test_datasets_provider.is_distributed = True + return train_valid_test_datasets_provider + elif args.model_type_name == "t5": + from pretrain_t5 import train_valid_test_datasets_provider + + train_valid_test_datasets_provider.is_distributed = True + return train_valid_test_datasets_provider + except ImportError: + pass + return train_valid_test_datasets_provider + + def build_train_valid_test_data_iterators(self, accelerator): + args = get_args() + + train_valid_test_dataset_provider = self.get_train_valid_test_datasets_provider(accelerator) + if args.virtual_pipeline_model_parallel_size is not None: + train_data_iterator = [] + valid_data_iterator = [] + test_data_iterator = [] + for i in range(getattr(args, "model_len", 0)): + mpu.set_virtual_pipeline_model_parallel_rank(i) + iterators = build_train_valid_test_data_iterators(train_valid_test_dataset_provider) + train_data_iterator.append(iterators[0]) + valid_data_iterator.append(iterators[1]) + test_data_iterator.append(iterators[2]) + else: + train_data_iterator, valid_data_iterator, test_data_iterator = build_train_valid_test_data_iterators( + train_valid_test_dataset_provider + ) + + return train_data_iterator, valid_data_iterator, test_data_iterator + + +def _handle_megatron_data_iterator(accelerator, data_iterator): + class DummyMegatronDataloader: + def __iter__(self): + return self + + def __next__(self): + return {} + + is_data_iterator_empty = data_iterator is None + is_src_data_iterator_empty = torch.tensor(is_data_iterator_empty, dtype=torch.bool, device=accelerator.device) + torch.distributed.broadcast( + is_src_data_iterator_empty, get_tensor_model_parallel_src_rank(), group=get_tensor_model_parallel_group() + ) + if not is_src_data_iterator_empty and is_data_iterator_empty: + return DummyMegatronDataloader() + return data_iterator + + +def prepare_data_loader(accelerator, dataloader): + accelerator.print("Preparing dataloader") + args = get_args() + if not args.megatron_dataset_flag: + from ..data_loader import _PYTORCH_DATALOADER_KWARGS, prepare_data_loader + + micro_batch_size = args.micro_batch_size * args.num_micro_batches + kwargs = {k: getattr(dataloader, k, _PYTORCH_DATALOADER_KWARGS[k]) for k in _PYTORCH_DATALOADER_KWARGS} + if kwargs["batch_size"] is None: + if isinstance(kwargs["sampler"], torch.utils.data.BatchSampler): + kwargs["sampler"].batch_size = micro_batch_size + else: + del kwargs["sampler"] + del kwargs["shuffle"] + del kwargs["batch_size"] + kwargs["batch_sampler"].batch_size = micro_batch_size + else: + del kwargs["batch_sampler"] + kwargs["batch_size"] = micro_batch_size + + dataloader = torch.utils.data.DataLoader(dataloader.dataset, **kwargs) + # split_batches: + # Megatron only needs to fetch different data between different dp groups, + # and does not need to split the data within the dp group. + return prepare_data_loader( + dataloader, + accelerator.device, + num_processes=mpu.get_data_parallel_world_size(), + process_index=mpu.get_data_parallel_rank(), + split_batches=False, + put_on_device=True, + rng_types=accelerator.rng_types.copy(), + dispatch_batches=accelerator.dispatch_batches, + ) + else: + if args.consumed_samples is not None: + ( + args.consumed_train_samples, + args.consumed_valid_samples, + args.consumed_test_samples, + ) = args.consumed_samples + else: + args.consumed_train_samples, args.consumed_valid_samples, args.consumed_test_samples = 0, 0, 0 + args.micro_batch_size = args.micro_batch_size * args.num_micro_batches + # In order to be compatible with data in transform format, + # it needs to increase the size of mbs first, + # and then split the large batch data into some mbs. + ( + train_data_iterator, + valid_data_iterator, + test_data_iterator, + ) = dataloader.build_train_valid_test_data_iterators(accelerator) + args.micro_batch_size = args.micro_batch_size // args.num_micro_batches + + train_data_iterator = _handle_megatron_data_iterator( + accelerator=accelerator, data_iterator=train_data_iterator + ) + valid_data_iterator = _handle_megatron_data_iterator( + accelerator=accelerator, data_iterator=valid_data_iterator + ) + test_data_iterator = _handle_megatron_data_iterator(accelerator=accelerator, data_iterator=test_data_iterator) + + return train_data_iterator, valid_data_iterator, test_data_iterator + + +# optimizer utilities +class MegatronLMOptimizerWrapper(AcceleratedOptimizer): + def __init__(self, optimizer): + super().__init__(optimizer, device_placement=False, scaler=None) + + def zero_grad(self, set_to_none=None): + pass # `model(**batch)` is doing that automatically. Therefore, its implementation is not needed + + def step(self): + pass # `model(**batch)` is doing that automatically. Therefore, its implementation is not needed + + @property + def step_was_skipped(self): + """Whether or not the optimizer step was done, or skipped because of gradient overflow.""" + return self.optimizer.skipped_iter + + +def prepare_optimizer(accelerator, model): + accelerator.print("Preparing optimizer") + args = get_args() + return get_megatron_optimizer(model, args.no_wd_decay_cond, args.scale_lr_cond, args.lr_mult) + + +# scheduler utilities +class MegatronLMDummyScheduler: + """ + Dummy scheduler presents model parameters or param groups, this is primarily used to follow conventional training + loop when scheduler config is specified in the deepspeed config file. + + Args: + optimizer (`torch.optim.optimizer.Optimizer`): + The optimizer to wrap. + total_num_steps (int): + Total number of steps. + warmup_num_steps (int): + Number of steps for warmup. + **kwargs (additional keyword arguments, *optional*): + Other arguments. + """ + + def __init__(self, optimizer, total_num_steps=None, warmup_num_steps=0, **kwargs): + self.optimizer = optimizer + self.total_num_steps = total_num_steps + self.warmup_num_steps = warmup_num_steps + self.kwargs = kwargs + + +class MegatronLMSchedulerWrapper(AcceleratedScheduler): + def __init__(self, scheduler, optimizers): + super().__init__(scheduler, optimizers) + + def step(self, *args, **kwargs): + return # `model(**batch)` is doing that automatically. Therefore, its implementation is not needed + + +def prepare_scheduler(accelerator, optimizer, scheduler): + accelerator.print("Preparing scheduler") + scheduler = get_optimizer_param_scheduler(optimizer) + return scheduler + + +class AbstractTrainStep(ABC): + """Abstract class for batching, forward pass and loss handler.""" + + def __init__(self, name): + super().__init__() + self.name = name + + def get_batch_func(self, accelerator, megatron_dataset_flag): + pass + + def get_forward_step_func(self): + pass + + def get_loss_func(self, accelerator): + pass + + +class BertTrainStep(AbstractTrainStep): + """ + Bert train step class. + + Args: + args (`argparse.Namespace`): Megatron-LM arguments. + """ + + def __init__(self, accelerator, args): + super().__init__("BertTrainStep") + self.get_batch = self.get_batch_func(accelerator, args.megatron_dataset_flag) + self.loss_func = self.get_loss_func(accelerator, args.pretraining_flag, args.num_labels) + self.forward_step = self.get_forward_step_func(args.pretraining_flag, args.bert_binary_head) + if not args.model_return_dict: + self.model_output_class = None + else: + from transformers.modeling_outputs import SequenceClassifierOutput + + self.model_output_class = SequenceClassifierOutput + + def get_batch_func(self, accelerator, megatron_dataset_flag): + def get_batch_megatron(data_iterator): + """Build the batch.""" + + # Items and their type. + keys = ["text", "types", "labels", "is_random", "loss_mask", "padding_mask"] + datatype = torch.int64 + + # Broadcast data. + if data_iterator is not None: + data = next(data_iterator) + else: + data = None + data_b = tensor_parallel.broadcast_data(keys, data, datatype) + + # Unpack. + tokens = data_b["text"].long() + types = data_b["types"].long() + sentence_order = data_b["is_random"].long() + loss_mask = data_b["loss_mask"].float() + lm_labels = data_b["labels"].long() + padding_mask = data_b["padding_mask"].long() + + return tokens, types, sentence_order, loss_mask, lm_labels, padding_mask + + def get_batch_transformer(data_iterator): + """Build the batch.""" + data = next(data_iterator) + data = send_to_device(data, torch.cuda.current_device()) + + # Unpack. + tokens = data["input_ids"].long() + padding_mask = data["attention_mask"].long() + if "token_type_ids" in data: + types = data["token_type_ids"].long() + else: + types = None + if "labels" in data: + lm_labels = data["labels"].long() + loss_mask = (data["labels"] != -100).to(torch.float) + else: + lm_labels = None + loss_mask = None + if "next_sentence_label" in data: + sentence_order = data["next_sentence_label"].long() + else: + sentence_order = None + + return tokens, types, sentence_order, loss_mask, lm_labels, padding_mask + + if accelerator.state.megatron_lm_plugin.custom_get_batch_function is not None: + return accelerator.state.megatron_lm_plugin.custom_get_batch_function + if megatron_dataset_flag: + try: + # Use '--no-use-pep517 -e' to pip install nvidia's megatron from source + from pretrain_bert import get_batch + + return get_batch + except ImportError: + pass + return get_batch_megatron + else: + return get_batch_transformer + + def get_loss_func(self, accelerator, pretraining_flag, num_labels): + def loss_func_pretrain(loss_mask, sentence_order, output_tensor): + lm_loss_, sop_logits = output_tensor + + lm_loss_ = lm_loss_.float() + loss_mask = loss_mask.float() + lm_loss = torch.sum(lm_loss_.view(-1) * loss_mask.reshape(-1)) / loss_mask.sum() + + if sop_logits is not None: + sop_loss = F.cross_entropy(sop_logits.view(-1, 2).float(), sentence_order.view(-1), ignore_index=-1) + sop_loss = sop_loss.float() + loss = lm_loss + sop_loss + averaged_losses = average_losses_across_data_parallel_group([lm_loss, sop_loss]) + return loss, {"lm loss": averaged_losses[0], "sop loss": averaged_losses[1]} + + else: + loss = lm_loss + averaged_losses = average_losses_across_data_parallel_group([lm_loss]) + return loss, {"lm loss": averaged_losses[0]} + + def loss_func_finetune(labels, logits): + if num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + elif self.num_labels > 1 and (labels.dtype in (torch.long, torch.int)): + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, num_labels), labels.view(-1)) + else: + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(logits, labels) + averaged_losses = average_losses_across_data_parallel_group([loss]) + return loss, {"loss": averaged_losses[0]} + + if accelerator.state.megatron_lm_plugin.custom_loss_function is not None: + return accelerator.state.megatron_lm_plugin.custom_loss_function + if pretraining_flag: + return loss_func_pretrain + else: + return loss_func_finetune + + def get_forward_step_func(self, pretraining_flag, bert_binary_head): + def forward_step(data_iterator, model): + """Forward step.""" + tokens, types, sentence_order, loss_mask, labels, padding_mask = self.get_batch(data_iterator) + if not bert_binary_head: + types = None + # Forward pass through the model. + if pretraining_flag: + output_tensor = model(tokens, padding_mask, tokentype_ids=types, lm_labels=labels) + return output_tensor, partial(self.loss_func, loss_mask, sentence_order) + else: + logits = model(tokens, padding_mask, tokentype_ids=types) + return logits, partial(self.loss_func, labels) + + return forward_step + + +class GPTTrainStep(AbstractTrainStep): + """ + GPT train step class. + + Args: + args (`argparse.Namespace`): Megatron-LM arguments. + """ + + def __init__(self, accelerator, args): + super().__init__("GPTTrainStep") + self.get_batch = self.get_batch_func(accelerator, args.megatron_dataset_flag) + self.loss_func = self.get_loss_func(accelerator) + self.forward_step = self.get_forward_step_func() + if args.vocab_file is not None: + tokenizer = get_tokenizer() + self.eod_token = tokenizer.eod + self.eod_token = args.eos_token_id + self.pad_token = args.eos_token_id + self.reset_position_ids = args.reset_position_ids + self.reset_attention_mask = args.reset_attention_mask + self.eod_mask_loss = args.eod_mask_loss + if not args.model_return_dict: + self.model_output_class = None + else: + from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions + + self.model_output_class = CausalLMOutputWithCrossAttentions + + def get_batch_func(self, accelerator, megatron_dataset_flag): + def get_batch_megatron(data_iterator): + """Generate a batch""" + # Items and their type. + keys = ["text"] + datatype = torch.int64 + + # Broadcast data. + if data_iterator is not None: + data = next(data_iterator) + else: + data = None + data_b = tensor_parallel.broadcast_data(keys, data, datatype) + + # Unpack. + tokens_ = data_b["text"].long() + labels = tokens_[:, 1:].contiguous() + tokens = tokens_[:, :-1].contiguous() + + # Get the masks and position ids. + attention_mask, loss_mask, position_ids = get_ltor_masks_and_position_ids( + tokens, + eod_token=self.eod_token, + pad_token=self.eod_token, + reset_position_ids=self.reset_position_ids, + reset_attention_mask=self.reset_attention_mask, + eod_mask_loss=self.eod_mask_loss, + pad_mask_loss=True, + ) + return tokens, labels, loss_mask, attention_mask, position_ids + + def get_batch_transformer(data_iterator): + data = next(data_iterator) + data = {"input_ids": data["input_ids"]} + data = send_to_device(data, torch.cuda.current_device()) + + tokens_ = data["input_ids"].long() + padding = torch.zeros((tokens_.shape[0], 1), dtype=tokens_.dtype, device=tokens_.device) + self.eod_token + tokens_ = torch.concat([tokens_, padding], dim=1) + labels = tokens_[:, 1:].contiguous() + tokens = tokens_[:, :-1].contiguous() + # Get the masks and position ids. + attention_mask, loss_mask, position_ids = get_ltor_masks_and_position_ids( + tokens, + eod_token=self.eod_token, + pad_token=self.eod_token, + reset_position_ids=self.reset_position_ids, + reset_attention_mask=self.reset_attention_mask, + eod_mask_loss=self.eod_mask_loss, + pad_mask_loss=True, + ) + return tokens, labels, loss_mask, attention_mask, position_ids + + if accelerator.state.megatron_lm_plugin.custom_get_batch_function is not None: + return accelerator.state.megatron_lm_plugin.custom_get_batch_function + if megatron_dataset_flag: + try: + # Use '--no-use-pep517 -e' to pip install nvidia's megatron from source + from pretrain_gpt import get_batch + + return get_batch + except ImportError: + pass + return get_batch_megatron + else: + return get_batch_transformer + + def get_loss_func(self, accelerator): + args = get_args() + + def loss_func(loss_mask, output_tensor): + if args.return_logits: + losses, logits = output_tensor + else: + losses = output_tensor + losses = losses.float() + loss_mask = loss_mask.view(-1).float() + if args.context_parallel_size > 1: + loss = torch.cat([torch.sum(losses.view(-1) * loss_mask).view(1), loss_mask.sum().view(1)]) + torch.distributed.all_reduce(loss, group=mpu.get_context_parallel_group()) + loss = loss[0] / loss[1] + else: + loss = torch.sum(losses.view(-1) * loss_mask) / loss_mask.sum() + + # Check individual rank losses are not NaN prior to DP all-reduce. + if args.check_for_nan_in_loss_and_grad: + global_rank = torch.distributed.get_rank() + assert not loss.isnan(), ( + f"Rank {global_rank}: found NaN in local forward loss calculation. " + f"Device: {torch.cuda.current_device()}, node: {os.uname()[1]}" + ) + + # Reduce loss for logging. + averaged_loss = average_losses_across_data_parallel_group([loss]) + + output_dict = {"lm loss": averaged_loss[0]} + if args.return_logits: + output_dict.update({"logits": logits}) + return loss, output_dict + + if accelerator.state.megatron_lm_plugin.custom_loss_function is not None: + return accelerator.state.megatron_lm_plugin.custom_loss_function + return loss_func + + def get_forward_step_func(self): + def forward_step(data_iterator, model): + """Forward step.""" + # Get the batch. + tokens, labels, loss_mask, attention_mask, position_ids = self.get_batch(data_iterator) + output_tensor = model(tokens, position_ids, attention_mask, labels=labels) + + return output_tensor, partial(self.loss_func, loss_mask) + + return forward_step + + +class T5TrainStep(AbstractTrainStep): + """ + T5 train step class. + + Args: + args (`argparse.Namespace`): Megatron-LM arguments. + """ + + def __init__(self, accelerator, args): + super().__init__("T5TrainStep") + self.get_batch = self.get_batch_func(accelerator, args.megatron_dataset_flag) + self.loss_func = self.get_loss_func(accelerator) + self.forward_step = self.get_forward_step_func() + if not args.model_return_dict: + self.model_output_class = None + else: + from transformers.modeling_outputs import Seq2SeqLMOutput + + self.model_output_class = Seq2SeqLMOutput + + @staticmethod + def attn_mask_postprocess(attention_mask): + # We create a 3D attention mask from a 2D tensor mask. + # [b, 1, s] + attention_mask_b1s = attention_mask.unsqueeze(1) + # [b, s, 1] + attention_mask_bs1 = attention_mask.unsqueeze(2) + # [b, s, s] + attention_mask_bss = attention_mask_b1s * attention_mask_bs1 + # Convert attention mask to binary: + extended_attention_mask = attention_mask_bss < 0.5 + return extended_attention_mask + + @staticmethod + def get_decoder_mask(seq_length, device): + attention_mask = torch.tril(torch.ones((1, seq_length, seq_length), device=device)) + attention_mask = attention_mask < 0.5 + return attention_mask + + @staticmethod + def get_enc_dec_mask(attention_mask, dec_seq_length, device): + batch_size, _ = attention_mask.shape + # We create a 3D attention mask from a 2D tensor mask. + # [b, 1, s] + attention_mask_b1s = attention_mask.unsqueeze(1) + # [b, s, 1] + attention_mask_bs1 = torch.ones((batch_size, dec_seq_length, 1), device=device) + attention_mask_bss = attention_mask_bs1 * attention_mask_b1s + extended_attention_mask = attention_mask_bss < 0.5 + return extended_attention_mask + + def get_batch_func(self, accelerator, megatron_dataset_flag): + def get_batch_megatron(data_iterator): + """Build the batch.""" + + keys = ["text_enc", "text_dec", "labels", "loss_mask", "enc_mask", "dec_mask", "enc_dec_mask"] + datatype = torch.int64 + + # Broadcast data. + if data_iterator is not None: + data = next(data_iterator) + else: + data = None + data_b = tensor_parallel.broadcast_data(keys, data, datatype) + + # Unpack. + tokens_enc = data_b["text_enc"].long() + tokens_dec = data_b["text_dec"].long() + labels = data_b["labels"].long() + loss_mask = data_b["loss_mask"].float() + + enc_mask = data_b["enc_mask"] < 0.5 + dec_mask = data_b["dec_mask"] < 0.5 + enc_dec_mask = data_b["enc_dec_mask"] < 0.5 + + return tokens_enc, tokens_dec, loss_mask, labels, enc_mask, dec_mask, enc_dec_mask + + def get_batch_transformer(data_iterator): + """Build the batch.""" + data = next(data_iterator) + data = send_to_device(data, torch.cuda.current_device()) + + tokens_enc = data["input_ids"].long() + labels = data["labels"].long() + loss_mask = (labels != -100).to(torch.float) + if "decoder_input_ids" in data: + tokens_dec = data["decoder_input_ids"].long() + else: + tokens_dec = labels.new_zeros(labels.shape, device=labels.device, dtype=torch.long) + tokens_dec[..., 1:] = labels[..., :-1].clone() + tokens_dec[..., 0] = 0 + tokens_dec.masked_fill_(tokens_dec == -100, 0) + enc_mask = T5TrainStep.attn_mask_postprocess(data["attention_mask"].long()) + dec_mask = T5TrainStep.get_decoder_mask(tokens_dec.shape[1], tokens_dec.device) + enc_dec_mask = T5TrainStep.get_enc_dec_mask( + data["attention_mask"].long(), tokens_dec.shape[1], tokens_dec.device + ) + + return tokens_enc, tokens_dec, loss_mask, labels, enc_mask, dec_mask, enc_dec_mask + + if accelerator.state.megatron_lm_plugin.custom_get_batch_function is not None: + return accelerator.state.megatron_lm_plugin.custom_get_batch_function + if megatron_dataset_flag: + try: + # Use '--no-use-pep517 -e' to pip install nvidia's megatron from source + from pretrain_t5 import get_batch + + return get_batch + except ImportError: + pass + return get_batch_megatron + else: + return get_batch_transformer + + def get_loss_func(self, accelerator): + def loss_func(loss_mask, output_tensor): + lm_loss_ = output_tensor.float() + lm_loss = torch.sum(lm_loss_.view(-1) * loss_mask.reshape(-1)) / loss_mask.sum() + + loss = lm_loss + averaged_losses = average_losses_across_data_parallel_group([lm_loss]) + + return loss, {"lm loss": averaged_losses[0]} + + if accelerator.state.megatron_lm_plugin.custom_loss_function is not None: + return accelerator.state.megatron_lm_plugin.custom_loss_function + return loss_func + + def get_forward_step_func(self): + def forward_step(data_iterator, model): + """Forward step.""" + # Get the batch. + tokens_enc, tokens_dec, loss_mask, lm_labels, enc_mask, dec_mask, enc_dec_mask = self.get_batch( + data_iterator + ) + # Forward model lm_labels + output_tensor = model( + tokens_enc, tokens_dec, enc_mask, dec_mask, enc_dec_mask, tokentype_ids=None, lm_labels=lm_labels + ) + + return output_tensor, partial(self.loss_func, loss_mask) + + return forward_step + + +def finish_mpu_init(): + # torch.distributed initialization + args = get_args() + # Pytorch distributed. + _initialize_distributed(None, None, None) + + # Random seeds for reproducibility. + if args.rank == 0: + print(f"> setting random seeds to {args.seed} ...") + _set_random_seed(args.seed, args.data_parallel_random_init) + + +# initialize megatron setup +def initialize(accelerator, extra_args_provider=None, args_defaults=None): + if args_defaults is None: + args_defaults = {} + accelerator.print("Initializing Megatron-LM") + assert torch.cuda.is_available(), "Megatron requires CUDA." + + # Parse arguments + args = parse_args(extra_args_provider, ignore_unknown_args=True) + + # Set defaults + for key, value in args_defaults.items(): + if getattr(args, key, None) is not None: + if args.rank == 0: + print( + f"WARNING: overriding default arguments for {key}:{getattr(args, key)} with {key}:{value}", + flush=True, + ) + setattr(args, key, value) + + if args.use_checkpoint_args or args_defaults.get("use_checkpoint_args", False): + assert args.load is not None, "--use-checkpoints-args requires --load argument" + load_args_from_checkpoint(args) + + validate_args(args) + + # set global args, build tokenizer, and set adlr-autoresume, + # tensorboard-writer, and timers. + set_global_variables(args, build_tokenizer=False) + + # Megatron's MPU is the master. Complete initialization right away. + finish_mpu_init() + + # Autoresume. + _init_autoresume() + + # Compile dependencies. + _compile_dependencies() + + # Set pytorch JIT layer fusion options and warmup JIT functions. + set_jit_fusion_options() + args = get_args() + if getattr(args, "padded_vocab_size", None) is None: + args.padded_vocab_size = _vocab_size_with_padding(args.orig_vocab_size, args) + if args.model_type_name == "bert" and args.pretraining_flag and args.num_labels == 2: + args.bert_binary_head = True + else: + args.bert_binary_head = False + args.iteration = 0 + + +class MegatronEngine(torch.nn.Module): + """ + Megatron-LM model wrapper + + Args: + accelerator (:class:`~accelerate.Accelerator`): The accelerator object to use. + model: Megatron-LM model + optimizer: Megatron-LM optimizer + lr_scheduler: Megatron-LM lr scheduler + """ + + def __init__(self, accelerator, model, optimizer, scheduler): + super().__init__() + self.module = model + self.base_model = model[0] + self.optimizer = optimizer + self.scheduler = scheduler + args = get_args() + if accelerator.state.megatron_lm_plugin.custom_train_step_class is not None: + self.train_step_handler = accelerator.state.megatron_lm_plugin.custom_train_step_class( + args, **accelerator.state.megatron_lm_plugin.custom_train_step_kwargs + ) + elif args.model_type_name == "bert": + self.train_step_handler = BertTrainStep(accelerator, args) + elif args.model_type_name == "gpt": + self.train_step_handler = GPTTrainStep(accelerator, args) + elif args.model_type_name == "t5": + self.train_step_handler = T5TrainStep(accelerator, args) + else: + raise ValueError(f"Unsupported model type: {args.model_type_name}") + self.optimizer.skipped_iter = False + + # Tracking loss. + self.total_loss_dict = {} + self.eval_total_loss_dict = {} + self.iteration = 0 + self.report_memory_flag = True + self.num_floating_point_operations_so_far = 0 + self.module_config = None + if args.tensorboard_dir is not None: + write_args_to_tensorboard() + + def get_module_config(self): + args = get_args() + config = get_model_config(self.module[0]) + # Setup some training config params + config.grad_scale_func = self.optimizer.scale_loss + if isinstance(self.module[0], LocalDDP) and args.overlap_grad_reduce: + assert config.no_sync_func is None, ( + "When overlap_grad_reduce is True, config.no_sync_func must be None; " + "a custom no_sync_func is not supported when overlapping grad-reduce" + ) + config.no_sync_func = [model_chunk.no_sync for model_chunk in self.module] + if len(self.module) == 1: + config.no_sync_func = config.no_sync_func[0] + if args.delay_grad_reduce: + config.grad_sync_func = [model_chunk.start_grad_sync for model_chunk in self.module] + if len(self.module) == 1: + config.grad_sync_func = config.grad_sync_func[0] + if args.overlap_param_gather and args.delay_param_gather: + config.param_sync_func = [ + lambda x: self.optimizer.finish_param_sync(model_index, x) for model_index in range(len(self.module)) + ] + if len(self.module) == 1: + config.param_sync_func = config.param_sync_func[0] + config.finalize_model_grads_func = finalize_model_grads + return config + + def train(self): + for model_module in self.module: + model_module.train() + + if self.module_config is None: + self.module_config = self.get_module_config() + + self.log_eval_results() + + def eval(self): + for model_module in self.module: + model_module.eval() + + if self.module_config is None: + self.module_config = self.get_module_config() + + def get_batch_data_iterator(self, batch_data): + args = get_args() + data_chunks = [] + if len(batch_data) > 0: + if args.num_micro_batches > 1: + for i in range(0, args.num_micro_batches): + data_chunks.append( + { + k: v[i * args.micro_batch_size : (i + 1) * args.micro_batch_size] + for k, v in batch_data.items() + } + ) + else: + data_chunks = [batch_data] + + if len(self.module) > 1: + batch_data_iterator = ( + [iter(data_chunks) for _ in range(len(self.module))] + if len(batch_data) > 0 + else [None] * len(self.module) + ) + else: + batch_data_iterator = iter(data_chunks) if len(batch_data) > 0 else None + return batch_data_iterator + + def train_step(self, **batch_data): + """ + Training step for Megatron-LM + + Args: + batch_data (:obj:`dict`): The batch data to train on. + """ + + batch_data_iterator = self.get_batch_data_iterator(batch_data) + + loss_reduced, skipped_iter, _, _, _, grad_norm, num_zeros_in_grad = train_step( + forward_step_func=self.train_step_handler.forward_step, + data_iterator=batch_data_iterator, + model=self.module, + optimizer=self.optimizer, + opt_param_scheduler=self.scheduler, + config=self.module_config, + forward_backward_func=get_forward_backward_func(), + ) + + self.optimizer.skipped_iter = skipped_iter == 1 + + return loss_reduced, skipped_iter, grad_norm, num_zeros_in_grad + + def eval_step(self, **batch_data): + """ + Evaluation step for Megatron-LM + + Args: + batch_data (:obj:`dict`): The batch data to evaluate on. + """ + + args = get_args() + batch_data_iterator = self.get_batch_data_iterator(batch_data) + forward_backward_func = get_forward_backward_func() + loss_dicts = forward_backward_func( + forward_step_func=self.train_step_handler.forward_step, + data_iterator=batch_data_iterator, + model=self.module, + num_microbatches=get_num_microbatches(), + seq_length=args.seq_length, + micro_batch_size=args.micro_batch_size, + forward_only=True, + ) + # Empty unused memory + if args.empty_unused_memory_level >= 1: + torch.cuda.empty_cache() + + args.consumed_valid_samples += ( + mpu.get_data_parallel_world_size() * args.micro_batch_size * get_num_microbatches() + ) + + if mpu.is_pipeline_last_stage(ignore_virtual=True): + # Average loss across microbatches. + loss_reduced = {} + for key in loss_dicts[0]: + losses_reduced_for_key = [x[key] for x in loss_dicts] + if len(losses_reduced_for_key[0].shape) == 0: + loss_reduced[key] = sum(losses_reduced_for_key) / len(losses_reduced_for_key) + else: + loss_reduced[key] = torch.concat(losses_reduced_for_key) + return loss_reduced + return {} + + def forward(self, **batch_data): + # During training, we use train_step() + # model(**batch_data) performs following operations by delegating it to `self.train_step`: + # 1. Prepare **batch_data for Tendor, Pipeline and Model Parallelism + # 2. Set grad to zero. + # 3. forward pass and backward pass using Pipeline Parallelism + # 4. Empty unused memory. + # 5. Reduce gradients. + # 6. Update parameters. + # 7. Gather params when using Distributed Optimizer (Data Parallelism). + # 8. Update learning rate if scheduler is specified. + # 9. Empty unused memory. + # 10. Average loss across microbatches and across DP ranks. + # + # During evaluation, we use eval_step() + args = get_args() + if self.module[0].training: + loss_dict, skipped_iter, grad_norm, num_zeros_in_grad = self.train_step(**batch_data) + self.iteration += 1 + batch_size = mpu.get_data_parallel_world_size() * args.micro_batch_size * get_num_microbatches() + args.consumed_train_samples += batch_size + self.num_floating_point_operations_so_far += num_floating_point_operations(args, batch_size) + if args.tensorboard_dir is not None: + # Logging. + loss_scale = self.optimizer.get_loss_scale().item() + params_norm = None + if args.log_params_norm: + params_norm = calc_params_l2_norm(self.model) + self.report_memory_flag = training_log( + loss_dict, + self.total_loss_dict, + self.optimizer.param_groups[0]["lr"], + self.iteration, + loss_scale, + self.report_memory_flag, + skipped_iter, + grad_norm, + params_norm, + num_zeros_in_grad, + ) + else: + loss_dict = self.eval_step(**batch_data) + if args.tensorboard_dir is not None: + for key in loss_dict: + self.eval_total_loss_dict[key] = ( + self.eval_total_loss_dict.get(key, torch.cuda.FloatTensor([0.0])) + loss_dict[key] + ) + self.eval_total_loss_dict[key + "_num_iters"] = self.eval_total_loss_dict.get( + key + "_num_iters", torch.cuda.FloatTensor([0.0]) + ) + torch.cuda.FloatTensor([1.0]) + + loss = torch.tensor(0.0, device=torch.cuda.current_device()) + for key in loss_dict: + if len(loss_dict[key].shape) == 0: + loss += loss_dict[key] + + logits = None + if "logits" in loss_dict: + logits = loss_dict["logits"] + if self.train_step_handler.model_output_class is not None: + return self.train_step_handler.model_output_class(loss=loss, logits=logits) + return loss + + def log_eval_results(self): + args = get_args() + if args.tensorboard_dir is None or self.iteration == 0: + return + args = get_args() + writer = get_tensorboard_writer() + string = f"validation loss at iteration {self.iteration} | " + for key in self.eval_total_loss_dict: + if key.endswith("_num_iters"): + continue + value = self.eval_total_loss_dict[key] / self.eval_total_loss_dict[key + "_num_iters"] + string += f"{key} value: {value} | " + ppl = math.exp(min(20, value.item())) + if args.pretraining_flag: + string += f"{key} PPL: {ppl} | " + if writer: + writer.add_scalar(f"{key} validation", value.item(), self.iteration) + if args.pretraining_flag: + writer.add_scalar(f"{key} validation ppl", ppl, self.iteration) + + length = len(string) + 1 + print_rank_last("-" * length) + print_rank_last(string) + print_rank_last("-" * length) + self.eval_total_loss_dict = {} + + def save_checkpoint(self, output_dir): + self.log_eval_results() + args = get_args() + args.save = output_dir + torch.distributed.barrier() + save_checkpoint( + self.iteration, + self.module, + self.optimizer, + self.scheduler, + num_floating_point_operations_so_far=self.num_floating_point_operations_so_far, + ) + torch.distributed.barrier() + + def load_checkpoint(self, input_dir): + args = get_args() + args.load = input_dir + args.consumed_train_samples = 0 + args.consumed_valid_samples = 0 + torch.distributed.barrier() + iteration, num_floating_point_operations_so_far = load_checkpoint(self.module, self.optimizer, self.scheduler) + torch.distributed.barrier() + self.iteration = iteration + self.num_floating_point_operations_so_far = num_floating_point_operations_so_far + if args.fp16 and self.iteration == 0: + self.optimizer.reload_model_params() + + +# other utilities +def avg_losses_across_data_parallel_group(losses): + """ + Average losses across data parallel group. + + Args: + losses (List[Tensor]): List of losses to average across data parallel group. + """ + + return average_losses_across_data_parallel_group(losses) + + +def gather_across_data_parallel_groups(tensor): + """ + Recursively gather tensor in a nested list/tuple/dictionary of tensors from data parallel ranks. + + Args: + tensor (nested list/tuple/dictionary of `torch.Tensor`): + The data to gather across data parallel ranks. + + """ + + def _gpu_gather_one(tensor): + if tensor.ndim == 0: + tensor = tensor.clone()[None] + output_tensors = [ + torch.empty_like(tensor) + for _ in range(torch.distributed.get_world_size(group=mpu.get_data_parallel_group())) + ] + torch.distributed.all_gather(output_tensors, tensor, group=mpu.get_data_parallel_group()) + return torch.cat(output_tensors, dim=0) + + return recursively_apply(_gpu_gather_one, tensor, error_on_other_type=True) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/memory.py b/kbidu_env/Lib/site-packages/accelerate/utils/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..a5fd255b95085c09de376d25222266a3731d101a --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/memory.py @@ -0,0 +1,188 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A collection of utilities for ensuring that training can always occur. Heavily influenced by the +[toma](https://github.com/BlackHC/toma) library. +""" + +import functools +import gc +import inspect +from typing import Optional + +import torch + +from .imports import ( + is_cuda_available, + is_hpu_available, + is_mlu_available, + is_mps_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_xpu_available, +) + + +def clear_device_cache(garbage_collection=False): + """ + Clears the device cache by calling `torch.{backend}.empty_cache`. Can also run `gc.collect()`, but do note that + this is a *considerable* slowdown and should be used sparingly. + """ + if garbage_collection: + gc.collect() + + if is_xpu_available(): + torch.xpu.empty_cache() + elif is_mlu_available(): + torch.mlu.empty_cache() + elif is_sdaa_available(): + torch.sdaa.empty_cache() + elif is_musa_available(): + torch.musa.empty_cache() + elif is_npu_available(): + torch.npu.empty_cache() + elif is_mps_available(min_version="2.0"): + torch.mps.empty_cache() + elif is_cuda_available(): + torch.cuda.empty_cache() + elif is_hpu_available(): + # torch.hpu.empty_cache() # not available on hpu as it reserves all device memory for the current process + pass + elif is_neuron_available(): + # Not sure it actually does something, but adding for consistency with other backends + torch.neuron.empty_cache() + + +def release_memory(*objects): + """ + Releases memory from `objects` by setting them to `None` and calls `gc.collect()` and `torch.cuda.empty_cache()`. + Returned objects should be reassigned to the same variables. + + Args: + objects (`Iterable`): + An iterable of objects + Returns: + A list of `None` objects to replace `objects` + + Example: + + ```python + >>> import torch + >>> from accelerate.utils import release_memory + + >>> a = torch.ones(1000, 1000).cuda() + >>> b = torch.ones(1000, 1000).cuda() + >>> a, b = release_memory(a, b) + ``` + """ + if not isinstance(objects, list): + objects = list(objects) + for i in range(len(objects)): + objects[i] = None + clear_device_cache(garbage_collection=True) + return objects + + +def should_reduce_batch_size(exception: Exception) -> bool: + """ + Checks if `exception` relates to CUDA out-of-memory, XPU out-of-memory, CUDNN not supported, or CPU out-of-memory + + Args: + exception (`Exception`): + An exception + """ + _statements = [ + " out of memory.", # OOM for CUDA, HIP, XPU + "cuDNN error: CUDNN_STATUS_NOT_SUPPORTED.", # CUDNN SNAFU + "DefaultCPUAllocator: can't allocate memory", # CPU OOM + "FATAL ERROR :: MODULE:PT_DEVMEM Allocation failed", # HPU OOM + ] + if isinstance(exception, RuntimeError) and len(exception.args) == 1: + return any(err in exception.args[0] for err in _statements) + return False + + +def find_executable_batch_size( + function: Optional[callable] = None, + starting_batch_size: int = 128, + reduce_batch_size_fn: Optional[callable] = None, +): + """ + A basic decorator that will try to execute `function`. If it fails from exceptions related to out-of-memory or + CUDNN, the batch size is multiplied by 0.9 and passed to `function` + + `function` must take in a `batch_size` parameter as its first argument. + + Args: + function (`callable`, *optional*): + A function to wrap + starting_batch_size (`int`, *optional*): + The batch size to try and fit into memory + reduce_batch_size_fn (`callable`, *optional*): + A function to determine the new batch size after an out-of-memory error. If not + provided, the batch size is multiplied by 0.9 on each failure. The function takes + no arguments and should return the new (reduced) batch size as an `int`. + + Example: + + ```python + >>> from accelerate.utils import find_executable_batch_size + + + >>> @find_executable_batch_size(starting_batch_size=128) + ... def train(batch_size, model, optimizer): + ... ... + + + >>> train(model, optimizer) + ``` + """ + if function is None: + return functools.partial(find_executable_batch_size, starting_batch_size=starting_batch_size) + + batch_size = starting_batch_size + if reduce_batch_size_fn is None: + + def reduce_batch_size_fn(): + nonlocal batch_size + batch_size = int(batch_size * 0.9) + return batch_size + + def decorator(*args, **kwargs): + nonlocal batch_size + clear_device_cache(garbage_collection=True) + params = list(inspect.signature(function).parameters.keys()) + # Guard against user error + if len(params) < (len(args) + 1): + arg_str = ", ".join([f"{arg}={value}" for arg, value in zip(params[1:], args[1:])]) + raise TypeError( + f"Batch size was passed into `{function.__name__}` as the first argument when called." + f"Remove this as the decorator already does so: `{function.__name__}({arg_str})`" + ) + while True: + if batch_size == 0: + raise RuntimeError("No executable batch size found, reached zero.") + try: + return function(batch_size, *args, **kwargs) + except Exception as e: + if should_reduce_batch_size(e): + clear_device_cache(garbage_collection=True) + batch_size = reduce_batch_size_fn() + else: + raise + + return decorator diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/modeling.py b/kbidu_env/Lib/site-packages/accelerate/utils/modeling.py new file mode 100644 index 0000000000000000000000000000000000000000..047bb8b4e8eb774d8fbb64dacde6df310f833876 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/modeling.py @@ -0,0 +1,2204 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import gc +import inspect +import json +import logging +import os +import re +import shutil +import tempfile +import warnings +from collections import OrderedDict, defaultdict +from typing import Optional, Union + +import torch +from torch import distributed as dist +from torch import nn + +from ..state import AcceleratorState +from .constants import SAFE_WEIGHTS_NAME, WEIGHTS_NAME +from .dataclasses import AutocastKwargs, CustomDtype, DistributedType +from .imports import ( + is_hpu_available, + is_mlu_available, + is_mps_available, + is_musa_available, + is_npu_available, + is_peft_available, + is_sdaa_available, + is_torch_xla_available, + is_xpu_available, +) +from .memory import clear_device_cache +from .offload import load_offloaded_weight, offload_weight, save_offload_index +from .tqdm import is_tqdm_available, tqdm +from .versions import is_torch_version + + +if is_npu_available(check_device=False): + import torch_npu # noqa: F401 + +if is_mlu_available(check_device=False): + import torch_mlu # noqa: F401 + +if is_sdaa_available(check_device=False): + import torch_sdaa # noqa: F401 + +if is_musa_available(check_device=False): + import torch_musa # noqa: F401 + +from safetensors import safe_open +from safetensors.torch import load_file as safe_load_file + + +WEIGHTS_INDEX_NAME = "pytorch_model.bin.index.json" + +logger = logging.getLogger(__name__) + + +def is_peft_model(model): + from .other import extract_model_from_parallel + + if is_peft_available(): + from peft import PeftModel + + return is_peft_available() and isinstance(extract_model_from_parallel(model), PeftModel) + + +def check_device_same(first_device, second_device): + """ + Utility method to check if two `torch` devices are similar. When dealing torch accelerator devices(e.g. cuda, xpu), + torch throws `False` for `torch.device("cuda") == torch.device("cuda:0")` whereas they should be the same + + Args: + first_device (`torch.device`): + First device to check + second_device (`torch.device`): + Second device to check + """ + if first_device.type != second_device.type: + return False + + if first_device.type != "cpu" and first_device.index is None: + # In case the first_device is an torch accelerator device(e.g. cuda, xpu) and have + # the index attribute set to `None`, default it to `0` + first_device = torch.device(first_device.type, index=0) + + if second_device.type != "cpu" and second_device.index is None: + # In case the second_device is an torch accelerator device(e.g. cuda, xpu) and have + # the index attribute set to `None`, default it to `0` + second_device = torch.device(second_device.type, index=0) + + return first_device == second_device + + +def convert_file_size_to_int(size: Union[int, str]): + """ + Converts a size expressed as a string with digits an unit (like `"5MB"`) to an integer (in bytes). + + Args: + size (`int` or `str`): The size to convert. Will be directly returned if an `int`. + + Example: + + ```py + >>> convert_file_size_to_int("1MiB") + 1048576 + ``` + """ + mem_size = -1 + err_msg = ( + f"`size` {size} is not in a valid format. Use an integer for bytes, or a string with an unit (like '5.0GB')." + ) + try: + if isinstance(size, int): + mem_size = size + elif size.upper().endswith("GIB"): + mem_size = int(float(size[:-3]) * (2**30)) + elif size.upper().endswith("MIB"): + mem_size = int(float(size[:-3]) * (2**20)) + elif size.upper().endswith("KIB"): + mem_size = int(float(size[:-3]) * (2**10)) + elif size.upper().endswith("GB"): + int_size = int(float(size[:-2]) * (10**9)) + mem_size = int_size // 8 if size.endswith("b") else int_size + elif size.upper().endswith("MB"): + int_size = int(float(size[:-2]) * (10**6)) + mem_size = int_size // 8 if size.endswith("b") else int_size + elif size.upper().endswith("KB"): + int_size = int(float(size[:-2]) * (10**3)) + mem_size = int_size // 8 if size.endswith("b") else int_size + except ValueError: + raise ValueError(err_msg) + + if mem_size < 0: + raise ValueError(err_msg) + return mem_size + + +def dtype_byte_size(dtype: torch.dtype): + """ + Returns the size (in bytes) occupied by one parameter of type `dtype`. + + Example: + + ```py + >>> dtype_byte_size(torch.float32) + 4 + ``` + """ + if dtype == torch.bool: + return 1 / 8 + elif dtype == CustomDtype.INT2: + return 1 / 4 + elif dtype == CustomDtype.INT4: + return 1 / 2 + elif dtype == CustomDtype.FP8: + return 1 + elif is_torch_version(">=", "2.1.0") and dtype in [ + getattr(torch, name) + for name in ( + "float8_e4m3fn", + "float8_e5m2", + "float8_e4m3fnuz", + "float8_e5m2fnuz", + "float8_e8m0fnu", + ) + if hasattr(torch, name) + ]: + return 1 + bit_search = re.search(r"[^\d](\d+)$", str(dtype)) + if bit_search is None: + raise ValueError(f"`dtype` is not a valid dtype: {dtype}.") + bit_size = int(bit_search.groups()[0]) + return bit_size // 8 + + +def id_tensor_storage(tensor: torch.Tensor) -> tuple[torch.device, int, int]: + """ + Unique identifier to a tensor storage. Multiple different tensors can share the same underlying storage. For + example, "meta" tensors all share the same storage, and thus their identifier will all be equal. This identifier is + guaranteed to be unique and constant for this tensor's storage during its lifetime. Two tensor storages with + non-overlapping lifetimes may have the same id. + """ + _SIZE = { + torch.int64: 8, + torch.float32: 4, + torch.int32: 4, + torch.bfloat16: 2, + torch.float16: 2, + torch.int16: 2, + torch.uint8: 1, + torch.int8: 1, + torch.bool: 1, + torch.float64: 8, + } + try: + storage_ptr = tensor.untyped_storage().data_ptr() + storage_size = tensor.untyped_storage().nbytes() + except Exception: + try: + # Fallback for torch==1.10 + storage_ptr = tensor.storage().data_ptr() + storage_size = tensor.storage().size() * _SIZE[tensor.dtype] + except NotImplementedError: + # Fallback for meta storage + storage_ptr = 0 + # On torch >=2.0 this is the tensor size + storage_size = tensor.nelement() * _SIZE[tensor.dtype] + + return tensor.device, storage_ptr, storage_size + + +def set_module_tensor_to_device( + module: nn.Module, + tensor_name: str, + device: Union[int, str, torch.device], + value: Optional[torch.Tensor] = None, + dtype: Optional[Union[str, torch.dtype]] = None, + fp16_statistics: Optional[torch.HalfTensor] = None, + tied_params_map: Optional[dict[int, dict[torch.device, torch.Tensor]]] = None, + non_blocking: bool = False, + clear_cache: bool = True, +): + """ + A helper function to set a given tensor (parameter of buffer) of a module on a specific device (note that doing + `param.to(device)` creates a new tensor not linked to the parameter, which is why we need this function). + + Args: + module (`torch.nn.Module`): + The module in which the tensor we want to move lives. + tensor_name (`str`): + The full name of the parameter/buffer. + device (`int`, `str` or `torch.device`): + The device on which to set the tensor. + value (`torch.Tensor`, *optional*): + The value of the tensor (useful when going from the meta device to any other device). + dtype (`torch.dtype`, *optional*): + If passed along the value of the parameter will be cast to this `dtype`. Otherwise, `value` will be cast to + the dtype of the existing parameter in the model. + fp16_statistics (`torch.HalfTensor`, *optional*): + The list of fp16 statistics to set on the module, used for 8 bit model serialization. + tied_params_map (Dict[int, Dict[torch.device, torch.Tensor]], *optional*, defaults to `None`): + A map of current data pointers to dictionaries of devices to already dispatched tied weights. For a given + execution device, this parameter is useful to reuse the first available pointer of a shared weight on the + device for all others, instead of duplicating memory. + non_blocking (`bool`, *optional*, defaults to `False`): + If `True`, the device transfer will be asynchronous with respect to the host, if possible. + clear_cache (`bool`, *optional*, defaults to `True`): + Whether or not to clear the device cache after setting the tensor on the device. + """ + # Recurse if needed + if "." in tensor_name: + splits = tensor_name.split(".") + for split in splits[:-1]: + new_module = getattr(module, split) + if new_module is None: + raise ValueError(f"{module} has no attribute {split}.") + module = new_module + tensor_name = splits[-1] + + if tensor_name not in module._parameters and tensor_name not in module._buffers: + raise ValueError(f"{module} does not have a parameter or a buffer named {tensor_name}.") + is_buffer = tensor_name in module._buffers + old_value = getattr(module, tensor_name) + + # Treat the case where old_value (or a custom `value`, typically offloaded to RAM/disk) belongs to a tied group, and one of the weight + # in the tied group has already been dispatched to the device, by avoiding reallocating memory on the device and just copying the pointer. + if ( + value is not None + and tied_params_map is not None + and value.data_ptr() in tied_params_map + and device in tied_params_map[value.data_ptr()] + ): + module._parameters[tensor_name] = tied_params_map[value.data_ptr()][device] + return + elif ( + tied_params_map is not None + and old_value.data_ptr() in tied_params_map + and device in tied_params_map[old_value.data_ptr()] + ): + module._parameters[tensor_name] = tied_params_map[old_value.data_ptr()][device] + return + + if old_value.device == torch.device("meta") and device not in ["meta", torch.device("meta")] and value is None: + raise ValueError(f"{tensor_name} is on the meta device, we need a `value` to put in on {device}.") + + param = module._parameters[tensor_name] if tensor_name in module._parameters else None + param_cls = type(param) + + if value is not None: + # We can expect mismatches when using bnb 4bit since Params4bit will reshape and pack the weights. + # In other cases, we want to make sure we're not loading checkpoints that do not match the config. + if old_value.shape != value.shape and param_cls.__name__ != "Params4bit": + raise ValueError( + f'Trying to set a tensor of shape {value.shape} in "{tensor_name}" (which has shape {old_value.shape}), this looks incorrect.' + ) + + if dtype is None: + # For compatibility with PyTorch load_state_dict which converts state dict dtype to existing dtype in model + value = value.to(old_value.dtype, non_blocking=non_blocking) + elif not str(value.dtype).startswith(("torch.uint", "torch.int", "torch.bool")): + value = value.to(dtype, non_blocking=non_blocking) + + device_quantization = None + with torch.no_grad(): + # leave it on cpu first before moving them to device + # # fix the case where the device is meta, we don't want to put it on cpu because there is no data =0 + if ( + param is not None + and param.device.type not in ("cuda", "xpu") + and torch.device(device).type in ("cuda", "xpu") + and param_cls.__name__ in ["Int8Params", "FP4Params", "Params4bit"] + ): + device_quantization = device + device = "cpu" + # `torch.Tensor.to()` is not supported by `torch_npu` (see this [issue](https://github.com/Ascend/pytorch/issues/16)). + if isinstance(device, int): + if is_npu_available(): + device = f"npu:{device}" + elif is_mlu_available(): + device = f"mlu:{device}" + elif is_sdaa_available(): + device = f"sdaa:{device}" + elif is_musa_available(): + device = f"musa:{device}" + elif is_hpu_available(): + device = "hpu" + if "xpu" in str(device) and not is_xpu_available(): + raise ValueError(f'{device} is not available, you should use device="cpu" instead') + if value is None: + new_value = old_value.to(device, non_blocking=non_blocking) + if dtype is not None and device in ["meta", torch.device("meta")]: + if not str(old_value.dtype).startswith(("torch.uint", "torch.int", "torch.bool")): + new_value = new_value.to(dtype, non_blocking=non_blocking) + + if not is_buffer: + module._parameters[tensor_name] = param_cls(new_value, requires_grad=old_value.requires_grad) + elif isinstance(value, torch.Tensor): + new_value = value.to(device, non_blocking=non_blocking) + else: + new_value = torch.tensor(value, device=device) + if device_quantization is not None: + device = device_quantization + if is_buffer: + module._buffers[tensor_name] = new_value + elif value is not None or not check_device_same(torch.device(device), module._parameters[tensor_name].device): + param_cls = type(module._parameters[tensor_name]) + kwargs = module._parameters[tensor_name].__dict__ + is_hf_initialized = kwargs.pop("_is_hf_initialized", None) + if param_cls.__name__ in ["Int8Params", "FP4Params", "Params4bit"]: + if param_cls.__name__ == "Int8Params" and new_value.dtype == torch.float32: + # downcast to fp16 if any - needed for 8bit serialization + new_value = new_value.to(torch.float16, non_blocking=non_blocking) + # quantize module that are going to stay on the cpu so that we offload quantized weights + if device == "cpu" and param_cls.__name__ == "Int8Params": + new_value = param_cls(new_value, requires_grad=old_value.requires_grad, **kwargs).to(0).to("cpu") + new_value.CB = new_value.CB.to("cpu") + new_value.SCB = new_value.SCB.to("cpu") + else: + new_value = param_cls(new_value, requires_grad=old_value.requires_grad, **kwargs).to( + device, non_blocking=non_blocking + ) + elif param_cls.__name__ in ["QTensor", "QBitsTensor"]: + new_value = torch.nn.Parameter(new_value, requires_grad=old_value.requires_grad).to( + device, non_blocking=non_blocking + ) + elif param_cls.__name__ in ["AffineQuantizedTensor"] or "torchao" in getattr(param_cls, "__module__", ""): + new_value = new_value.to(device, non_blocking=non_blocking) + else: + new_value = param_cls(new_value, requires_grad=old_value.requires_grad).to( + device, non_blocking=non_blocking + ) + + if is_hf_initialized is not None: + new_value._is_hf_initialized = is_hf_initialized + module._parameters[tensor_name] = new_value + if fp16_statistics is not None: + module._parameters[tensor_name].SCB = fp16_statistics.to(device, non_blocking=non_blocking) + del fp16_statistics + # as we put the weight to meta, it doesn't have SCB attr anymore. make sure that it is not a meta weight + if ( + module.__class__.__name__ == "Linear8bitLt" + and getattr(module.weight, "SCB", None) is None + and str(module.weight.device) != "meta" + ): + # quantize only if necessary + device_index = torch.device(device).index if torch.device(device).type in ["cuda", "xpu"] else None + if not getattr(module.weight, "SCB", None) and device_index is not None: + if module.bias is not None and module.bias.device.type != "meta": + # if a bias exists, we need to wait until the bias is set on the correct device + module = module.to(device_index) + elif module.bias is None: + # if no bias exists, we can quantize right away + module = module.to(device_index) + elif ( + module.__class__.__name__ == "Linear4bit" + and getattr(module.weight, "quant_state", None) is None + and str(module.weight.device) != "meta" + ): + # quantize only if necessary + device_index = torch.device(device).index if torch.device(device).type in ["cuda", "xpu"] else None + if not getattr(module.weight, "quant_state", None) and device_index is not None: + module.weight = module.weight.to(device_index) + + # clean pre and post forward hook + if clear_cache and device not in ("cpu", "meta"): + clear_device_cache() + + # When handling tied weights, we update tied_params_map to keep track of the tied weights that have already been allocated on the device in + # order to avoid duplicating memory, see above. + if ( + tied_params_map is not None + and old_value.data_ptr() in tied_params_map + and device not in tied_params_map[old_value.data_ptr()] + ): + tied_params_map[old_value.data_ptr()][device] = new_value + elif ( + value is not None + and tied_params_map is not None + and value.data_ptr() in tied_params_map + and device not in tied_params_map[value.data_ptr()] + ): + tied_params_map[value.data_ptr()][device] = new_value + + +def named_module_tensors( + module: nn.Module, include_buffers: bool = True, recurse: bool = False, remove_non_persistent: bool = False +): + """ + A helper function that gathers all the tensors (parameters + buffers) of a given module. If `include_buffers=True` + it's the same as doing `module.named_parameters(recurse=recurse) + module.named_buffers(recurse=recurse)`. + + Args: + module (`torch.nn.Module`): + The module we want the tensors on. + include_buffer (`bool`, *optional*, defaults to `True`): + Whether or not to include the buffers in the result. + recurse (`bool`, *optional`, defaults to `False`): + Whether or not to go look in every submodule or just return the direct parameters and buffers. + remove_non_persistent (`bool`, *optional*, defaults to `False`): + Whether or not to remove the non persistent buffer from the buffers. Useful only when include_buffers = + True + """ + yield from module.named_parameters(recurse=recurse) + + if include_buffers: + non_persistent_buffers = set() + if remove_non_persistent: + non_persistent_buffers = get_non_persistent_buffers(module, recurse=recurse) + for named_buffer in module.named_buffers(recurse=recurse): + name, _ = named_buffer + if name not in non_persistent_buffers: + yield named_buffer + + +def get_non_persistent_buffers(module: nn.Module, recurse: bool = False, fqns: bool = False): + """ + Gather all non persistent buffers of a given modules into a set + + Args: + module (`nn.Module`): + The module we want the non persistent buffers on. + recurse (`bool`, *optional*, defaults to `False`): + Whether or not to go look in every submodule or just return the direct non persistent buffers. + fqns (`bool`, *optional*, defaults to `False`): + Whether or not to return the fully-qualified names of the non persistent buffers. + """ + + non_persistent_buffers_set = module._non_persistent_buffers_set + if recurse: + for n, m in module.named_modules(): + if fqns: + non_persistent_buffers_set |= {n + "." + b for b in m._non_persistent_buffers_set} + else: + non_persistent_buffers_set |= m._non_persistent_buffers_set + + return non_persistent_buffers_set + + +def check_tied_parameters_in_config(model: nn.Module): + """ + Check if there is any indication in the given model that some weights should be tied. + + Args: + model (`torch.nn.Module`): The model to inspect + + Returns: + bool: True if the model needs to have tied weights + """ + + # based on model.tie_weights() method + has_tied_word_embedding = False + has_tied_encoder_decoder = False + has_tied_module = False + + if "PreTrainedModel" in [c.__name__ for c in inspect.getmro(model.__class__)]: + has_tied_word_embedding = False + model_decoder_config = None + if hasattr(model, "config"): + model_decoder_config = ( + model.config.get_text_config(decoder=True) + if hasattr(model.config, "get_text_config") + else model.config + ) + has_tied_word_embedding = ( + model_decoder_config is not None + and getattr(model_decoder_config, "tie_word_embeddings", False) + and model.get_output_embeddings() + ) + + has_tied_encoder_decoder = ( + hasattr(model, "config") + and getattr(model.config, "is_encoder_decoder", False) + and getattr(model.config, "tie_encoder_decoder", False) + ) + has_tied_module = any(hasattr(module, "_tie_weights") for module in model.modules()) + return any([has_tied_word_embedding, has_tied_encoder_decoder, has_tied_module]) + + +def _get_param_device(param, device_map): + if param in device_map: + return device_map[param] + parent_param = ".".join(param.split(".")[:-1]) + if parent_param == param: + raise ValueError(f"The `device_map` does not contain the module {param}.") + else: + return _get_param_device(parent_param, device_map) + + +def check_tied_parameters_on_same_device(tied_params, device_map): + """ + Check if tied parameters are on the same device + + Args: + tied_params (`List[List[str]]`): + A list of lists of parameter names being all tied together. + + device_map (`Dict[str, Union[int, str, torch.device]]`): + A map that specifies where each submodule should go. + + """ + for tie_param in tied_params: + tie_param_devices = {} + for param in tie_param: + tie_param_devices[param] = _get_param_device(param, device_map) + if len(set(tie_param_devices.values())) > 1: + logger.warning( + f"Tied parameters are on different devices: {tie_param_devices}. " + "Please modify your custom device map or set `device_map='auto'`. " + ) + + +def find_tied_parameters(model: torch.nn.Module, **kwargs) -> list[list[str]]: + """ + Find the tied parameters in a given model. + + + + The signature accepts keyword arguments, but they are for the recursive part of this function and you should ignore + them. + + + + Args: + model (`torch.nn.Module`): The model to inspect. + + Returns: + List[List[str]]: A list of lists of parameter names being all tied together. + + Example: + + ```py + >>> from collections import OrderedDict + >>> import torch.nn as nn + + >>> model = nn.Sequential(OrderedDict([("linear1", nn.Linear(4, 4)), ("linear2", nn.Linear(4, 4))])) + >>> model.linear2.weight = model.linear1.weight + >>> find_tied_parameters(model) + [['linear1.weight', 'linear2.weight']] + ``` + """ + + # get ALL model parameters and their names + all_named_parameters = {name: param for name, param in model.named_parameters(remove_duplicate=False)} + + # get ONLY unique named parameters, + # if parameter is tied and have multiple names, it will be included only once + no_duplicate_named_parameters = {name: param for name, param in model.named_parameters(remove_duplicate=True)} + + # the difference of the two sets will give us the tied parameters + tied_param_names = set(all_named_parameters.keys()) - set(no_duplicate_named_parameters.keys()) + + # 'tied_param_names' contains the names of parameters that are tied in the model, but we do not know + # which names refer to the same parameter. To identify this, we need to group them together. + tied_param_groups = {} + for tied_param_name in tied_param_names: + tied_param = all_named_parameters[tied_param_name] + for param_name, param in no_duplicate_named_parameters.items(): + # compare if parameters are the same, if so, group their names together + if param is tied_param: + if param_name not in tied_param_groups: + tied_param_groups[param_name] = [] + tied_param_groups[param_name].append(tied_param_name) + + return [sorted([weight] + list(set(tied))) for weight, tied in tied_param_groups.items()] + + +def retie_parameters(model, tied_params): + """ + Reties tied parameters in a given model if the link was broken (for instance when adding hooks). + + Args: + model (`torch.nn.Module`): + The model in which to retie parameters. + tied_params (`List[List[str]]`): + A mapping parameter name to tied parameter name as obtained by `find_tied_parameters`. + """ + for tied_group in tied_params: + param_to_tie = None + # two loops : the first one to set param_to_tie , the second one to change the values of tied_group + for param_name in tied_group: + module = model + splits = param_name.split(".") + for split in splits[:-1]: + module = getattr(module, split) + param = getattr(module, splits[-1]) + if param_to_tie is None and param.device != torch.device("meta"): + param_to_tie = param + break + if param_to_tie is not None: + for param_name in tied_group: + module = model + splits = param_name.split(".") + for split in splits[:-1]: + module = getattr(module, split) + setattr(module, splits[-1], param_to_tie) + + +def _get_proper_dtype(dtype: Union[str, torch.device]) -> torch.dtype: + """ + Just does torch.dtype(dtype) if necessary. + """ + if isinstance(dtype, str): + # We accept "torch.float16" or just "float16" + dtype = dtype.replace("torch.", "") + dtype = getattr(torch, dtype) + return dtype + + +def compute_module_sizes( + model: nn.Module, + dtype: Optional[Union[str, torch.device]] = None, + special_dtypes: Optional[dict[str, Union[str, torch.device]]] = None, + buffers_only: bool = False, +): + """ + Compute the size of each submodule of a given model. + """ + if dtype is not None: + dtype = _get_proper_dtype(dtype) + dtype_size = dtype_byte_size(dtype) + if special_dtypes is not None: + special_dtypes = {key: _get_proper_dtype(dtyp) for key, dtyp in special_dtypes.items()} + special_dtypes_size = {key: dtype_byte_size(dtyp) for key, dtyp in special_dtypes.items()} + module_sizes = defaultdict(int) + + module_list = [] + + if not buffers_only: + module_list = named_module_tensors(model, recurse=True) + else: + module_list = model.named_buffers(recurse=True) + + for name, tensor in module_list: + if special_dtypes is not None and name in special_dtypes: + size = tensor.numel() * special_dtypes_size[name] + elif dtype is None: + size = tensor.numel() * dtype_byte_size(tensor.dtype) + elif str(tensor.dtype).startswith(("torch.uint", "torch.int", "torch.bool")): + # According to the code in set_module_tensor_to_device, these types won't be converted + # so use their original size here + size = tensor.numel() * dtype_byte_size(tensor.dtype) + else: + size = tensor.numel() * min(dtype_size, dtype_byte_size(tensor.dtype)) + name_parts = name.split(".") + for idx in range(len(name_parts) + 1): + module_sizes[".".join(name_parts[:idx])] += size + + return module_sizes + + +def compute_module_total_buffer_size( + model: nn.Module, + dtype: Optional[Union[str, torch.device]] = None, + special_dtypes: Optional[dict[str, Union[str, torch.device]]] = None, +): + """ + Compute the total size of buffers in each submodule of a given model. + """ + module_sizes = compute_module_sizes(model, dtype=dtype, special_dtypes=special_dtypes, buffers_only=True) + return module_sizes.get("", 0) + + +def get_max_layer_size( + modules: list[tuple[str, torch.nn.Module]], module_sizes: dict[str, int], no_split_module_classes: list[str] +): + """ + Utility function that will scan a list of named modules and return the maximum size used by one full layer. The + definition of a layer being: + - a module with no direct children (just parameters and buffers) + - a module whose class name is in the list `no_split_module_classes` + + Args: + modules (`List[Tuple[str, torch.nn.Module]]`): + The list of named modules where we want to determine the maximum layer size. + module_sizes (`Dict[str, int]`): + A dictionary mapping each layer name to its size (as generated by `compute_module_sizes`). + no_split_module_classes (`List[str]`): + A list of class names for layers we don't want to be split. + + Returns: + `Tuple[int, List[str]]`: The maximum size of a layer with the list of layer names realizing that maximum size. + """ + max_size = 0 + layer_names = [] + modules_to_treat = modules.copy() + while len(modules_to_treat) > 0: + module_name, module = modules_to_treat.pop(0) + modules_children = list(module.named_children()) if isinstance(module, torch.nn.Module) else [] + if len(modules_children) == 0 or module.__class__.__name__ in no_split_module_classes: + # No splitting this one so we compare to the max_size + size = module_sizes[module_name] + if size > max_size: + max_size = size + layer_names = [module_name] + elif size == max_size: + layer_names.append(module_name) + else: + modules_to_treat = [(f"{module_name}.{n}", v) for n, v in modules_children] + modules_to_treat + return max_size, layer_names + + +def get_max_memory(max_memory: Optional[dict[Union[int, str], Union[int, str]]] = None): + """ + Get the maximum memory available if nothing is passed, converts string to int otherwise. + """ + import psutil + + if max_memory is None: + max_memory = {} + # Make sure device is initialized on each device to have the right memory info. + if is_npu_available(): + for i in range(torch.npu.device_count()): + try: + _ = torch.tensor(0, device=torch.device("npu", i)) + max_memory[i] = torch.npu.mem_get_info(i)[0] + except Exception: + logger.info(f"Device {i} seems unavailable, Proceeding to check subsequent devices.") + continue + elif is_mlu_available(): + for i in range(torch.mlu.device_count()): + try: + _ = torch.tensor(0, device=torch.device("mlu", i)) + max_memory[i] = torch.mlu.mem_get_info(i)[0] + except Exception: + logger.info(f"Device {i} seems unavailable, Proceeding to check subsequent devices.") + continue + elif is_sdaa_available(): + for i in range(torch.sdaa.device_count()): + try: + _ = torch.tensor(0, device=torch.device("sdaa", i)) + max_memory[i] = torch.sdaa.mem_get_info(i)[0] + except Exception: + logger.info(f"Device {i} seems unavailable, Proceeding to check subsequent devices.") + continue + elif is_musa_available(): + for i in range(torch.musa.device_count()): + try: + _ = torch.tensor(0, device=torch.device("musa", i)) + max_memory[i] = torch.musa.mem_get_info(i)[0] + except Exception: + logger.info(f"Device {i} seems unavailable, Proceeding to check subsequent devices.") + continue + elif is_xpu_available(): + for i in range(torch.xpu.device_count()): + try: + _ = torch.tensor(0, device=torch.device("xpu", i)) + max_memory[i] = torch.xpu.mem_get_info(i)[0] + except Exception: + logger.info(f"Device {i} seems unavailable, Proceeding to check subsequent devices.") + continue + elif is_hpu_available(): + for i in range(torch.hpu.device_count()): + try: + _ = torch.tensor(0, device=torch.device("hpu", i)) + max_memory[i] = torch.hpu.mem_get_info(i)[0] + except Exception: + logger.info(f"Device {i} seems unavailable, Proceeding to check subsequent devices.") + continue + else: + for i in range(torch.cuda.device_count()): + try: + _ = torch.tensor([0], device=i) + max_memory[i] = torch.cuda.mem_get_info(i)[0] + except Exception: + logger.info(f"Device {i} seems unavailable, Proceeding to check subsequent devices.") + continue + # allocate everything in the mps device as the RAM is shared + if is_mps_available(): + max_memory["mps"] = psutil.virtual_memory().available + else: + max_memory["cpu"] = psutil.virtual_memory().available + return max_memory + + for key in max_memory: + if isinstance(max_memory[key], str): + max_memory[key] = convert_file_size_to_int(max_memory[key]) + + # Need to sort the device by type to make sure that we allocate the gpu first. + # As gpu/npu/xpu are represented by int, we need to sort them first. + gpu_devices = [k for k in max_memory.keys() if isinstance(k, int)] + gpu_devices.sort() + # check if gpu/npu/xpu devices are available and if not, throw a warning + if is_npu_available(): + num_devices = torch.npu.device_count() + elif is_mlu_available(): + num_devices = torch.mlu.device_count() + elif is_sdaa_available(): + num_devices = torch.sdaa.device_count() + elif is_musa_available(): + num_devices = torch.musa.device_count() + elif is_xpu_available(): + num_devices = torch.xpu.device_count() + elif is_hpu_available(): + num_devices = torch.hpu.device_count() + else: + num_devices = torch.cuda.device_count() + for device in gpu_devices: + if device >= num_devices or device < 0: + logger.warning(f"Device {device} is not available, available devices are {list(range(num_devices))}") + # Add the other devices in the preset order if they are available + all_devices = gpu_devices + [k for k in ["mps", "cpu", "disk"] if k in max_memory.keys()] + # Raise an error if a device is not recognized + for k in max_memory.keys(): + if k not in all_devices: + raise ValueError( + f"Device {k} is not recognized, available devices are integers(for GPU/XPU), 'mps', 'cpu' and 'disk'" + ) + max_memory = {k: max_memory[k] for k in all_devices} + + return max_memory + + +def clean_device_map(device_map: dict[str, Union[int, str, torch.device]], module_name: str = ""): + """ + Cleans a device_map by grouping all submodules that go on the same device together. + """ + # Get the value of the current module and if there is only one split across several keys, regroup it. + prefix = "" if module_name == "" else f"{module_name}." + values = [v for k, v in device_map.items() if k.startswith(prefix)] + if len(set(values)) == 1 and len(values) > 1: + for k in [k for k in device_map if k.startswith(prefix)]: + del device_map[k] + device_map[module_name] = values[0] + + # Recurse over the children + children_modules = [k for k in device_map.keys() if k.startswith(prefix) and len(k) > len(module_name)] + idx = len(module_name.split(".")) + 1 if len(module_name) > 0 else 1 + children_modules = set(".".join(k.split(".")[:idx]) for k in children_modules) + for child in children_modules: + clean_device_map(device_map, module_name=child) + + return device_map + + +def load_offloaded_weights(model, index, offload_folder): + """ + Loads the weights from the offload folder into the model. + + Args: + model (`torch.nn.Module`): + The model to load the weights into. + index (`dict`): + A dictionary containing the parameter name and its metadata for each parameter that was offloaded from the + model. + offload_folder (`str`): + The folder where the offloaded weights are stored. + """ + if index is None or len(index) == 0: + # Nothing to do + return + for param_name, metadata in index.items(): + if "SCB" in param_name: + continue + fp16_statistics = None + if "weight" in param_name and param_name.replace("weight", "SCB") in index.keys(): + weight_name = param_name.replace("weight", "SCB") + fp16_statistics = load_offloaded_weight( + os.path.join(offload_folder, f"{weight_name}.dat"), index[weight_name] + ) + tensor_file = os.path.join(offload_folder, f"{param_name}.dat") + weight = load_offloaded_weight(tensor_file, metadata) + set_module_tensor_to_device(model, param_name, "cpu", value=weight, fp16_statistics=fp16_statistics) + + +def get_module_leaves(module_sizes): + module_children = {} + for module in module_sizes: + if module == "" or "." not in module: + continue + parent = module.rsplit(".", 1)[0] + module_children[parent] = module_children.get(parent, 0) + 1 + leaves = [module for module in module_sizes if module_children.get(module, 0) == 0 and module != ""] + return leaves + + +def get_balanced_memory( + model: nn.Module, + max_memory: Optional[dict[Union[int, str], Union[int, str]]] = None, + no_split_module_classes: Optional[list[str]] = None, + dtype: Optional[Union[str, torch.dtype]] = None, + special_dtypes: Optional[dict[str, Union[str, torch.device]]] = None, + low_zero: bool = False, +): + """ + Compute a `max_memory` dictionary for [`infer_auto_device_map`] that will balance the use of each available GPU. + + + + All computation is done analyzing sizes and dtypes of the model parameters. As a result, the model can be on the + meta device (as it would if initialized within the `init_empty_weights` context manager). + + + + Args: + model (`torch.nn.Module`): + The model to analyze. + max_memory (`Dict`, *optional*): + A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset. + Example: `max_memory={0: "1GB"}`. + no_split_module_classes (`List[str]`, *optional*): + A list of layer class names that should never be split across device (for instance any layer that has a + residual connection). + dtype (`str` or `torch.dtype`, *optional*): + If provided, the weights will be converted to that type when loaded. + special_dtypes (`Dict[str, Union[str, torch.device]]`, *optional*): + If provided, special dtypes to consider for some specific weights (will override dtype used as default for + all weights). + low_zero (`bool`, *optional*): + Minimizes the number of weights on GPU 0, which is convenient when it's used for other operations (like the + Transformers generate function). + """ + # Get default / clean up max_memory + user_not_set_max_memory = max_memory is None + max_memory = get_max_memory(max_memory) + + if is_npu_available(): + expected_device_type = "npu" + elif is_mlu_available(): + expected_device_type = "mlu" + elif is_sdaa_available(): + expected_device_type = "sdaa" + elif is_musa_available(): + expected_device_type = "musa" + elif is_xpu_available(): + expected_device_type = "xpu" + elif is_hpu_available(): + expected_device_type = "hpu" + elif is_mps_available(): + expected_device_type = "mps" + else: + expected_device_type = "cuda" + num_devices = len([d for d in max_memory if torch.device(d).type == expected_device_type and max_memory[d] > 0]) + + if num_devices == 0: + return max_memory + + if num_devices == 1: + # We cannot do low_zero on just one GPU, but we will still reserve some memory for the buffer + low_zero = False + # If user just asked us to handle memory usage, we should avoid OOM + if user_not_set_max_memory: + for key in max_memory.keys(): + if isinstance(key, int): + max_memory[key] *= 0.9 # 90% is a good compromise + logger.info( + f"We will use 90% of the memory on device {key} for storing the model, and 10% for the buffer to avoid OOM. " + "You can set `max_memory` in to a higher value to use more memory (at your own risk)." + ) + break # only one device + + module_sizes = compute_module_sizes(model, dtype=dtype, special_dtypes=special_dtypes) + per_gpu = module_sizes[""] // (num_devices - 1 if low_zero else num_devices) + + # We can't just set the memory to model_size // num_devices as it will end being too small: each GPU will get + # slightly less layers and some layers will end up offload at the end. So this function computes a buffer size to + # add which is the biggest of: + # - the size of no split block (if applicable) + # - the mean of the layer sizes + if no_split_module_classes is None: + no_split_module_classes = [] + elif isinstance(no_split_module_classes, str): + no_split_module_classes = [no_split_module_classes] + else: + no_split_module_classes = list(no_split_module_classes) + + # Identify the size of the no_split_block modules + if len(no_split_module_classes) > 0: + no_split_children = {} + for name, size in module_sizes.items(): + if name == "": + continue + submodule = model + for submodule_name in name.split("."): + submodule = getattr(submodule, submodule_name) + class_name = submodule.__class__.__name__ + if class_name in no_split_module_classes and class_name not in no_split_children: + no_split_children[class_name] = size + + if set(no_split_children.keys()) == set(no_split_module_classes): + break + buffer = max(no_split_children.values()) if len(no_split_children) > 0 else 0 + else: + buffer = 0 + + # Compute mean of final modules. In the first dict of module sizes, leaves are the parameters + leaves = get_module_leaves(module_sizes) + leaves_set = set(leaves) # Convert to set for O(1) membership testing + module_sizes = {n: v for n, v in module_sizes.items() if n not in leaves_set} + # Once removed, leaves are the final modules. + leaves = get_module_leaves(module_sizes) + mean_leaves = int(sum([module_sizes[n] for n in leaves]) / max(len(leaves), 1)) + buffer = int(1.25 * max(buffer, mean_leaves)) + per_gpu += buffer + + # Sorted list of GPUs id (we may have some gpu ids not included in the our max_memory list - let's ignore them) + gpus_idx_list = list( + sorted( + device_id for device_id, device_mem in max_memory.items() if isinstance(device_id, int) and device_mem > 0 + ) + ) + # The last device is left with max_memory just in case the buffer is not enough. + for idx in gpus_idx_list[:-1]: + max_memory[idx] = min(max_memory[0] if low_zero and idx == 0 else per_gpu, max_memory[idx]) + + if low_zero: + min_zero = max(0, module_sizes[""] - sum([max_memory[i] for i in range(1, num_devices)])) + max_memory[0] = min(min_zero, max_memory[0]) + + return max_memory + + +def calculate_maximum_sizes(model: torch.nn.Module): + "Computes the total size of the model and its largest layer" + sizes = compute_module_sizes(model) + # `transformers` models store this information for us + no_split_modules = getattr(model, "_no_split_modules", None) + if no_split_modules is None: + no_split_modules = [] + + modules_to_treat = ( + list(model.named_parameters(recurse=False)) + + list(model.named_children()) + + list(model.named_buffers(recurse=False)) + ) + largest_layer = get_max_layer_size(modules_to_treat, sizes, no_split_modules) + total_size = sizes[""] + return total_size, largest_layer + + +def _init_infer_auto_device_map( + model: nn.Module, + max_memory: Optional[dict[Union[int, str], Union[int, str]]] = None, + no_split_module_classes: Optional[list[str]] = None, + dtype: Optional[Union[str, torch.dtype]] = None, + special_dtypes: Optional[dict[str, Union[str, torch.device]]] = None, +) -> tuple[ + list[Union[int, str]], + dict[Union[int, str], Union[int, str]], + list[Union[int, str]], + list[int], + dict[str, int], + list[list[str]], + list[str], + list[tuple[str, nn.Module]], +]: + """ + Initialize variables required for computing the device map for model allocation. + """ + max_memory = get_max_memory(max_memory) + if no_split_module_classes is None: + no_split_module_classes = [] + elif isinstance(no_split_module_classes, str): + no_split_module_classes = [no_split_module_classes] + else: + no_split_module_classes = list(no_split_module_classes) + + devices = list(max_memory.keys()) + if "disk" not in devices: + devices.append("disk") + gpus = [device for device in devices if device not in ["cpu", "disk"]] + + # Devices that need to keep space for a potential offloaded layer. + if "mps" in gpus: + main_devices = ["mps"] + elif len(gpus) > 0: + main_devices = [gpus[0], "cpu"] + else: + main_devices = ["cpu"] + + module_sizes = compute_module_sizes(model, dtype=dtype, special_dtypes=special_dtypes) + tied_parameters = find_tied_parameters(model) + if check_tied_parameters_in_config(model) and len(tied_parameters) == 0: + logger.warning( + "The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function." + ) + + # Direct submodules and parameters + modules_to_treat = ( + list(model.named_parameters(recurse=False)) + + list(model.named_children()) + + list(model.named_buffers(recurse=False)) + ) + + return ( + devices, + max_memory, + main_devices, + gpus, + module_sizes, + tied_parameters, + no_split_module_classes, + modules_to_treat, + ) + + +def get_module_size_with_ties( + tied_params, + module_size, + module_sizes, + modules_to_treat, +) -> tuple[int, list[str], list[nn.Module]]: + """ + Calculate the total size of a module, including its tied parameters. + + Args: + tied_params (`List[str]`): The list of tied parameters. + module_size (`int`): The size of the module without tied parameters. + module_sizes (`Dict[str, int]`): A dictionary mapping each layer name to its size. + modules_to_treat (`List[Tuple[str, nn.Module]]`): The list of named modules to treat. + + Returns: + `Tuple[int, List[str], List[nn.Module]]`: The total size of the module, the names of the tied modules, and the + tied modules. + """ + if len(tied_params) < 1: + return module_size, [], [] + tied_module_names = [] + tied_modules = [] + + for tied_param in tied_params: + tied_module_index = [i for i, (n, _) in enumerate(modules_to_treat) if tied_param.startswith(n + ".")][0] + tied_module_names.append(modules_to_treat[tied_module_index][0]) + tied_modules.append(modules_to_treat[tied_module_index][1]) + + module_size_with_ties = module_size + for tied_param, tied_module_name in zip(tied_params, tied_module_names): + module_size_with_ties += module_sizes[tied_module_name] - module_sizes[tied_param] + + return module_size_with_ties, tied_module_names, tied_modules + + +def fallback_allocate( + modules: list[tuple[str, nn.Module]], + module_sizes: dict[str, int], + size_limit: Union[int, str], + no_split_module_classes: Optional[list[str]] = None, + tied_parameters: Optional[list[list[str]]] = None, +) -> tuple[Optional[str], Optional[nn.Module], list[tuple[str, nn.Module]]]: + """ + Find a module that fits in the size limit using BFS and return it with its name and the remaining modules. + + Args: + modules (`List[Tuple[str, nn.Module]]`): + The list of named modules to search in. + module_sizes (`Dict[str, int]`): + A dictionary mapping each layer name to its size (as generated by `compute_module_sizes`). + size_limit (`Union[int, str]`): + The maximum size a module can have. + no_split_module_classes (`Optional[List[str]]`, *optional*): + A list of class names for layers we don't want to be split. + tied_parameters (`Optional[List[List[str]]`, *optional*): + A list of lists of parameter names being all tied together. + + Returns: + `Tuple[Optional[str], Optional[nn.Module], List[Tuple[str, nn.Module]]]`: A tuple containing: + - The name of the module that fits within the size limit. + - The module itself. + - The list of remaining modules after the found module is removed. + """ + try: + size_limit = convert_file_size_to_int(size_limit) + except ValueError: + return None, None, modules + + if no_split_module_classes is None: + no_split_module_classes = [] + + if tied_parameters is None: + tied_parameters = [] + + modules_to_search = modules.copy() + module_found = False + + while modules_to_search: + name, module = modules_to_search.pop(0) + + tied_param_groups = [ + tied_group + for tied_group in tied_parameters + if any(name + "." in k + "." for k in tied_group) and not all(name + "." in k + "." for k in tied_group) + ] + + tied_params = sum( + [[p for p in tied_group if name + "." not in p + "."] for tied_group in tied_param_groups], [] + ) + + module_size_with_ties, _, _ = get_module_size_with_ties( + tied_params, module_sizes[name], module_sizes, modules_to_search + ) + + # If the module fits in the size limit, we found it. + if module_size_with_ties <= size_limit: + module_found = True + break + + # The module is too big, we need to split it if possible. + modules_children = ( + [] + if isinstance(module, nn.Parameter) or isinstance(module, torch.Tensor) + else list(module.named_children()) + ) + + # Split fails, move to the next module + if len(modules_children) == 0 or module.__class__.__name__ in no_split_module_classes: + continue + + # split is possible, add the children to the list of modules to search + modules_children = list(module.named_parameters(recurse=False)) + modules_children + modules_to_search = [(f"{name}.{n}", v) for n, v in modules_children] + modules_to_search + + if not module_found: + return None, None, modules + + # Prepare the module list for removal of the found module + current_names = [n for n, _ in modules] + dot_idx = [i for i, c in enumerate(name) if c == "."] + + for dot_index in dot_idx: + parent_name = name[:dot_index] + if parent_name in current_names: + parent_module_idx = current_names.index(parent_name) + _, parent_module = modules[parent_module_idx] + module_children = list(parent_module.named_parameters(recurse=False)) + list( + parent_module.named_children() + ) + modules = ( + modules[:parent_module_idx] + + [(f"{parent_name}.{n}", v) for n, v in module_children] + + modules[parent_module_idx + 1 :] + ) + current_names = [n for n, _ in modules] + + # Now the target module should be directly in the list + target_idx = current_names.index(name) + name, module = modules.pop(target_idx) + + return name, module, modules + + +def infer_auto_device_map( + model: nn.Module, + max_memory: Optional[dict[Union[int, str], Union[int, str]]] = None, + no_split_module_classes: Optional[list[str]] = None, + dtype: Optional[Union[str, torch.dtype]] = None, + special_dtypes: Optional[dict[str, Union[str, torch.dtype]]] = None, + verbose: bool = False, + clean_result: bool = True, + offload_buffers: bool = False, + fallback_allocation: bool = False, +): + """ + Compute a device map for a given model giving priority to GPUs, then offload on CPU and finally offload to disk, + such that: + - we don't exceed the memory available of any of the GPU. + - if offload to the CPU is needed, there is always room left on GPU 0 to put back the layer offloaded on CPU that + has the largest size. + - if offload to the CPU is needed,we don't exceed the RAM available on the CPU. + - if offload to the disk is needed, there is always room left on the CPU to put back the layer offloaded on disk + that has the largest size. + + + + All computation is done analyzing sizes and dtypes of the model parameters. As a result, the model can be on the + meta device (as it would if initialized within the `init_empty_weights` context manager). + + + + Args: + model (`torch.nn.Module`): + The model to analyze. + max_memory (`Dict`, *optional*): + A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset. + Example: `max_memory={0: "1GB"}`. + no_split_module_classes (`List[str]`, *optional*): + A list of layer class names that should never be split across device (for instance any layer that has a + residual connection). + dtype (`str` or `torch.dtype`, *optional*): + If provided, the weights will be converted to that type when loaded. + special_dtypes (`Dict[str, Union[str, torch.device]]`, *optional*): + If provided, special dtypes to consider for some specific weights (will override dtype used as default for + all weights). + verbose (`bool`, *optional*, defaults to `False`): + Whether or not to provide debugging statements as the function builds the device_map. + clean_result (`bool`, *optional*, defaults to `True`): + Clean the resulting device_map by grouping all submodules that go on the same device together. + offload_buffers (`bool`, *optional*, defaults to `False`): + In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as + well as the parameters. + fallback_allocation (`bool`, *optional*, defaults to `False`): + When regular allocation fails, try to allocate a module that fits in the size limit using BFS. + """ + + # Initialize the variables + ( + devices, + max_memory, + main_devices, + gpus, + module_sizes, + tied_parameters, + no_split_module_classes, + modules_to_treat, + ) = _init_infer_auto_device_map(model, max_memory, no_split_module_classes, dtype, special_dtypes) + + device_map = OrderedDict() + current_device = 0 + device_memory_used = {device: 0 for device in devices} + device_buffer_sizes = {} + device_minimum_assignment_memory = {} + + # Initialize maximum largest layer, to know which space to keep in memory + max_layer_size, max_layer_names = get_max_layer_size(modules_to_treat, module_sizes, no_split_module_classes) + + # Ready ? This is going to be a bit messy. + while len(modules_to_treat) > 0: + name, module = modules_to_treat.pop(0) + if verbose: + print(f"\nTreating module {name}.") + # Max size in the remaining layers may have changed since we took one, so we maybe update it. + max_layer_names = [n for n in max_layer_names if n != name and not n.startswith(name + ".")] + if len(max_layer_names) == 0: + max_layer_size, max_layer_names = get_max_layer_size( + [(n, m) for n, m in modules_to_treat if isinstance(m, torch.nn.Module)], + module_sizes, + no_split_module_classes, + ) + # Assess size needed + module_size = module_sizes[name] + + # We keep relevant tied parameters only: one of the tied parameters in the group is inside the current module + # and the other is not. + # Note: If we are currently processing the name `compute.weight`, an other parameter named + # e.g. `compute.weight_submodule.parameter` + # needs to be considered outside the current module, hence the check with additional dots. + tied_param_groups = [ + tied_group + for tied_group in tied_parameters + if any(name + "." in k + "." for k in tied_group) and not all(name + "." in k + "." for k in tied_group) + ] + + if verbose and len(tied_param_groups) > 0: + print(f" Found the relevant tied param groups {tied_param_groups}") + + # Then we keep track of all the parameters that are tied to the current module, but not in the current module + tied_params = sum( + [[p for p in tied_group if name + "." not in p + "."] for tied_group in tied_param_groups], [] + ) + + if verbose and len(tied_params) > 0: + print(f" So those parameters need to be taken into account {tied_params}") + + device = devices[current_device] + current_max_size = max_memory[device] if device != "disk" else None + current_memory_reserved = 0 + # Reduce max size available by the largest layer. + if devices[current_device] in main_devices: + current_max_size = current_max_size - max_layer_size + current_memory_reserved = max_layer_size + + module_size_with_ties, tied_module_names, tied_modules = get_module_size_with_ties( + tied_params, module_size, module_sizes, modules_to_treat + ) + + # The module and its tied modules fit on the current device. + if current_max_size is None or device_memory_used[device] + module_size_with_ties <= current_max_size: + if verbose: + output = f"Putting {name}" + + if tied_module_names: + output += f" and {tied_module_names}" + else: + output += f" (size={module_size})" + + if current_max_size is not None: + output += f" (available={current_max_size - device_memory_used[device]})" + + output += f" on {device}." + print(output) + + device_memory_used[device] += module_size_with_ties + + # Assign the primary module to the device. + device_map[name] = device + + # Assign tied modules if any. + for tied_module_name in tied_module_names: + if tied_module_name in [m[0] for m in modules_to_treat]: + # Find the index of the tied module in the list + tied_module_index = next(i for i, (n, _) in enumerate(modules_to_treat) if n == tied_module_name) + # Remove the tied module from the list to prevent reprocessing + modules_to_treat.pop(tied_module_index) + + # Assign the tied module to the device + device_map[tied_module_name] = device + + # Buffer Handling + if not offload_buffers and isinstance(module, nn.Module): + # Compute the total buffer size for the module + current_buffer_size = compute_module_total_buffer_size( + module, dtype=dtype, special_dtypes=special_dtypes + ) + # Update the buffer size on the device + device_buffer_sizes[device] = device_buffer_sizes.get(device, 0) + current_buffer_size + + continue + + # The current module itself fits, so we try to split the tied modules. + if len(tied_params) > 0 and device_memory_used[device] + module_size <= current_max_size: + # can we split one of the tied modules to make it smaller or do we need to go on the next device? + if verbose: + print( + f"Not enough space on {devices[current_device]} to put {name} and {tied_module_names} (space " + f"available {current_max_size - device_memory_used[device]}, needed size {module_size_with_ties})." + ) + split_happened = False + for tied_module_name, tied_module in zip(tied_module_names, tied_modules): + tied_module_children = list(tied_module.named_children()) + if len(tied_module_children) == 0 or tied_module.__class__.__name__ in no_split_module_classes: + # can't break this one. + continue + + if verbose: + print(f"Splitting {tied_module_name}.") + tied_module_children = list(tied_module.named_parameters(recurse=False)) + tied_module_children + tied_module_children = [(f"{tied_module_name}.{n}", v) for n, v in tied_module_children] + tied_module_index = [i for i, (n, _) in enumerate(modules_to_treat) if n == tied_module_name][0] + + modules_to_treat = ( + [(name, module)] + + modules_to_treat[:tied_module_index] + + tied_module_children + + modules_to_treat[tied_module_index + 1 :] + ) + # Update the max layer size. + max_layer_size, max_layer_names = get_max_layer_size( + [(n, m) for n, m in modules_to_treat if isinstance(m, torch.nn.Module)], + module_sizes, + no_split_module_classes, + ) + split_happened = True + break + + if split_happened: + continue + + # If the tied module is not split, we go to the next device + if verbose: + print("None of the tied module can be split, going to the next device.") + + # The current module itself doesn't fit, so we have to split it or go to the next device. + if device_memory_used[device] + module_size >= current_max_size: + # Split or not split? + modules_children = ( + [] + if isinstance(module, nn.Parameter) or isinstance(module, torch.Tensor) + else list(module.named_children()) + ) + if verbose: + print( + f"Not enough space on {devices[current_device]} to put {name} (space available " + f"{current_max_size - device_memory_used[device]}, module size {module_size})." + ) + if len(modules_children) == 0 or module.__class__.__name__ in no_split_module_classes: + # -> no split, we go to the next device + if verbose: + print("This module cannot be split, going to the next device.") + + else: + # -> split, we replace the module studied by its children + parameters + if verbose: + print(f"Splitting {name}.") + modules_children = list(module.named_parameters(recurse=False)) + modules_children + modules_to_treat = [(f"{name}.{n}", v) for n, v in modules_children] + modules_to_treat + # Update the max layer size. + max_layer_size, max_layer_names = get_max_layer_size( + [(n, m) for n, m in modules_to_treat if isinstance(m, torch.nn.Module)], + module_sizes, + no_split_module_classes, + ) + continue + + # If no module is assigned to the current device, we attempt to allocate a fallback module + # if fallback_allocation is enabled. + if device_memory_used[device] == 0 and fallback_allocation and device != "disk": + # We try to allocate a module that fits in the size limit using BFS. + # Recompute the current max size as we need to consider the current module as well. + current_max_size = max_memory[device] - max(max_layer_size, module_size_with_ties) + + fallback_module_name, fallback_module, remaining_modules = fallback_allocate( + modules_to_treat, + module_sizes, + current_max_size - device_memory_used[device], + no_split_module_classes, + tied_parameters, + ) + # use the next iteration to put the fallback module on the next device to avoid code duplication + if fallback_module is not None: + modules_to_treat = [(fallback_module_name, fallback_module)] + [(name, module)] + remaining_modules + continue + + if device_memory_used[device] == 0: + device_minimum_assignment_memory[device] = module_size_with_ties + current_memory_reserved + + # Neither the current module nor any tied modules can be split, so we move to the next device. + device_memory_used[device] = device_memory_used[device] + current_memory_reserved + current_device += 1 + modules_to_treat = [(name, module)] + modules_to_treat + + device_memory_used = {device: mem for device, mem in device_memory_used.items() if mem > 0} + + if clean_result: + device_map = clean_device_map(device_map) + + non_gpu_buffer_size = device_buffer_sizes.get("cpu", 0) + device_buffer_sizes.get("disk", 0) + if non_gpu_buffer_size > 0 and not offload_buffers: + is_buffer_fit_any_gpu = False + for gpu_device, gpu_max_memory in max_memory.items(): + if gpu_device == "cpu" or gpu_device == "disk": + continue + + if not is_buffer_fit_any_gpu: + gpu_memory_used = device_memory_used.get(gpu_device, 0) + + if gpu_max_memory >= non_gpu_buffer_size + gpu_memory_used: + is_buffer_fit_any_gpu = True + + if len(gpus) > 0 and not is_buffer_fit_any_gpu: + warnings.warn( + f"Current model requires {non_gpu_buffer_size} bytes of buffer for offloaded layers, which seems does " + f"not fit any GPU's remaining memory. If you are experiencing a OOM later, please consider using " + f"offload_buffers=True." + ) + + if device_minimum_assignment_memory: + devices_info = "\n".join( + f" - {device}: {mem} bytes required" for device, mem in device_minimum_assignment_memory.items() + ) + logger.info( + f"Based on the current allocation process, no modules could be assigned to the following devices due to " + f"insufficient memory:\n" + f"{devices_info}\n" + f"These minimum requirements are specific to this allocation attempt and may vary. Consider increasing " + f"the available memory for these devices to at least the specified minimum, or adjusting the model config." + ) + return device_map + + +def check_device_map(model: nn.Module, device_map: dict[str, Union[int, str, torch.device]]): + """ + Checks a device map covers everything in a given model. + + Args: + model (`torch.nn.Module`): The model to check the device map against. + device_map (`Dict[str, Union[int, str, torch.device]]`): The device map to check. + """ + all_module_names = dict(model.named_modules()) + invalid_keys = [k for k in device_map if k != "" and k not in all_module_names] + + if invalid_keys: + warnings.warn( + f"The following device_map keys do not match any submodules in the model: {invalid_keys}", UserWarning + ) + + all_model_tensors = [name for name, _ in model.state_dict().items()] + for module_name in device_map.keys(): + if module_name == "": + all_model_tensors.clear() + break + else: + all_model_tensors = [ + name + for name in all_model_tensors + if not name == module_name and not name.startswith(module_name + ".") + ] + if len(all_model_tensors) > 0: + non_covered_params = ", ".join(all_model_tensors) + raise ValueError( + f"The device_map provided does not give any device for the following parameters: {non_covered_params}" + ) + + +def load_state_dict(checkpoint_file, device_map=None): + """ + Load a checkpoint from a given file. If the checkpoint is in the safetensors format and a device map is passed, the + weights can be fast-loaded directly on the GPU. + + Args: + checkpoint_file (`str`): The path to the checkpoint to load. + device_map (`Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer + name, once a given module name is inside, every submodule of it will be sent to the same device. + """ + if checkpoint_file.endswith(".safetensors"): + with safe_open(checkpoint_file, framework="pt") as f: + metadata = f.metadata() + weight_names = f.keys() + + if metadata is None: + logger.warning( + f"The safetensors archive passed at {checkpoint_file} does not contain metadata. " + "Make sure to save your model with the `save_pretrained` method. Defaulting to 'pt' metadata." + ) + metadata = {"format": "pt"} + + if metadata.get("format") not in ["pt", "tf", "flax"]: + raise OSError( + f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure " + "you save your model with the `save_pretrained` method." + ) + elif metadata["format"] != "pt": + raise ValueError(f"The checkpoint passed was saved with {metadata['format']}, we need a the pt format.") + if device_map is None: + return safe_load_file(checkpoint_file) + else: + # if we only have one device we can load everything directly + if len(set(device_map.values())) == 1: + device = list(device_map.values())[0] + target_device = device + if isinstance(device, int): + if is_npu_available(): + target_device = f"npu:{device}" + elif is_hpu_available(): + target_device = "hpu" + + return safe_load_file(checkpoint_file, device=target_device) + + devices = list(set(device_map.values()) - {"disk"}) + # cpu device should always exist as fallback option + if "cpu" not in devices: + devices.append("cpu") + + # For each device, get the weights that go there + device_weights = {device: [] for device in devices} + for module_name, device in device_map.items(): + if device in devices: + device_weights[device].extend( + [k for k in weight_names if k == module_name or k.startswith(module_name + ".")] + ) + + # all weights that haven't defined a device should be loaded on CPU + device_weights["cpu"].extend([k for k in weight_names if k not in sum(device_weights.values(), [])]) + tensors = {} + if is_tqdm_available(): + progress_bar = tqdm( + main_process_only=False, + total=sum([len(device_weights[device]) for device in devices]), + unit="w", + smoothing=0, + leave=False, + ) + else: + progress_bar = None + for device in devices: + target_device = device + if isinstance(device, int): + if is_npu_available(): + target_device = f"npu:{device}" + elif is_hpu_available(): + target_device = "hpu" + + with safe_open(checkpoint_file, framework="pt", device=target_device) as f: + for key in device_weights[device]: + if progress_bar is not None: + progress_bar.set_postfix(dev=device, refresh=False) + progress_bar.set_description(key) + tensors[key] = f.get_tensor(key) + if progress_bar is not None: + progress_bar.update() + if progress_bar is not None: + progress_bar.close() + + return tensors + else: + return torch.load(checkpoint_file, map_location=torch.device("cpu"), weights_only=True) + + +def get_state_dict_offloaded_model(model: nn.Module): + """ + Returns the state dictionary for an offloaded model via iterative onloading + + Args: + model (`torch.nn.Module`): + The offloaded model we want to save + """ + + state_dict = {} + placeholders = set() + for name, module in model.named_modules(): + if name == "": + continue + + try: + with align_module_device(module, "cpu"): + module_state_dict = module.state_dict() + except MemoryError: + raise MemoryError("Offloaded module must fit in CPU memory to call save_model!") from None + + for key in module_state_dict: + # ignore placeholder parameters that are still on the meta device + if module_state_dict[key].device == torch.device("meta"): + placeholders.add(name + f".{key}") + continue + params = module_state_dict[key] + state_dict[name + f".{key}"] = params.to("cpu") # move buffers to cpu + for key in placeholders.copy(): + if key in state_dict: + placeholders.remove(key) + if placeholders: + logger.warning(f"The following tensors were not saved because they were still on meta device: {placeholders}") + + return state_dict + + +def get_state_dict_from_offload( + module: nn.Module, + module_name: str, + state_dict: dict[str, Union[str, torch.tensor]], + device_to_put_offload: Union[int, str, torch.device] = "cpu", +): + """ + Retrieve the state dictionary (with parameters) from an offloaded module and load into a specified device (defaults + to cpu). + + Args: + module: (`torch.nn.Module`): + The module we want to retrieve a state dictionary from + module_name: (`str`): + The name of the module of interest + state_dict (`Dict[str, Union[int, str, torch.device]]`): + Dictionary of {module names: parameters} + device_to_put_offload (`Union[int, str, torch.device]`): + Device to load offloaded parameters into, defaults to the cpu. + """ + + root = module_name[: module_name.rfind(".")] # module name without .weight or .bias + + # do not move parameters if the module is not offloaded + if not has_offloaded_params(module): + device_to_put_offload = None + + # assign the device to which the offloaded parameters will be sent + with align_module_device(module, device_to_put_offload): + for m_key, params in module.state_dict().items(): + if (root + f".{m_key}") in state_dict: + state_dict[root + f".{m_key}"] = params + + return state_dict + + +def load_checkpoint_in_model( + model: nn.Module, + checkpoint: Union[str, os.PathLike], + device_map: Optional[dict[str, Union[int, str, torch.device]]] = None, + offload_folder: Optional[Union[str, os.PathLike]] = None, + dtype: Optional[Union[str, torch.dtype]] = None, + offload_state_dict: bool = False, + offload_buffers: bool = False, + keep_in_fp32_modules: Optional[list[str]] = None, + offload_8bit_bnb: bool = False, + strict: bool = False, + full_state_dict: bool = True, + broadcast_from_rank0: bool = False, +): + """ + Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are + loaded. + + + + Once loaded across devices, you still need to call [`dispatch_model`] on your model to make it able to run. To + group the checkpoint loading and dispatch in one single call, use [`load_checkpoint_and_dispatch`]. + + + + Args: + model (`torch.nn.Module`): + The model in which we want to load a checkpoint. + checkpoint (`str` or `os.PathLike`): + The folder checkpoint to load. It can be: + - a path to a file containing a whole model state dict + - a path to a `.json` file containing the index to a sharded checkpoint + - a path to a folder containing a unique `.index.json` file and the shards of a checkpoint. + - a path to a folder containing a unique pytorch_model.bin or a model.safetensors file. + device_map (`Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer + name, once a given module name is inside, every submodule of it will be sent to the same device. + offload_folder (`str` or `os.PathLike`, *optional*): + If the `device_map` contains any value `"disk"`, the folder where we will offload weights. + dtype (`str` or `torch.dtype`, *optional*): + If provided, the weights will be converted to that type when loaded. + offload_state_dict (`bool`, *optional*, defaults to `False`): + If `True`, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if + the weight of the CPU state dict + the biggest shard does not fit. + offload_buffers (`bool`, *optional*, defaults to `False`): + Whether or not to include the buffers in the weights offloaded to disk. + keep_in_fp32_modules(`List[str]`, *optional*): + A list of the modules that we keep in `torch.float32` dtype. + offload_8bit_bnb (`bool`, *optional*): + Whether or not to enable offload of 8-bit modules on cpu/disk. + strict (`bool`, *optional*, defaults to `False`): + Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model's + state_dict. + full_state_dict (`bool`, *optional*, defaults to `True`): if this is set to `True`, all the tensors in the + loaded state_dict will be gathered. No ShardedTensor and DTensor will be in the loaded state_dict. + broadcast_from_rank0 (`False`, *optional*, defaults to `False`): when the option is `True`, a distributed + `ProcessGroup` must be initialized. rank0 should receive a full state_dict and will broadcast the tensors + in the state_dict one by one to other ranks. Other ranks will receive the tensors and shard (if applicable) + according to the local shards in the model. + + """ + if offload_8bit_bnb: + from .bnb import quantize_and_offload_8bit + + tied_params = find_tied_parameters(model) + + if check_tied_parameters_in_config(model) and len(tied_params) == 0: + logger.warning( + "The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function." + ) + if device_map is not None: + check_tied_parameters_on_same_device(tied_params, device_map) + + if offload_folder is None and device_map is not None and "disk" in device_map.values(): + raise ValueError( + "At least one of the model submodule will be offloaded to disk, please pass along an `offload_folder`." + ) + elif offload_folder is not None and device_map is not None and "disk" in device_map.values(): + os.makedirs(offload_folder, exist_ok=True) + + if isinstance(dtype, str): + # We accept "torch.float16" or just "float16" + dtype = dtype.replace("torch.", "") + dtype = getattr(torch, dtype) + + checkpoint_files = None + index_filename = None + if os.path.isfile(checkpoint): + if str(checkpoint).endswith(".json"): + index_filename = checkpoint + else: + checkpoint_files = [checkpoint] + elif os.path.isdir(checkpoint): + # check if the whole state dict is present + potential_state_bin = [f for f in os.listdir(checkpoint) if f == WEIGHTS_NAME] + potential_state_safetensor = [f for f in os.listdir(checkpoint) if f == SAFE_WEIGHTS_NAME] + if len(potential_state_bin) == 1: + checkpoint_files = [os.path.join(checkpoint, potential_state_bin[0])] + elif len(potential_state_safetensor) == 1: + checkpoint_files = [os.path.join(checkpoint, potential_state_safetensor[0])] + else: + # otherwise check for sharded checkpoints + potential_index = [f for f in os.listdir(checkpoint) if f.endswith(".index.json")] + if len(potential_index) == 0: + raise ValueError( + f"{checkpoint} is not a folder containing a `.index.json` file or a {WEIGHTS_NAME} or a {SAFE_WEIGHTS_NAME} file" + ) + elif len(potential_index) == 1: + index_filename = os.path.join(checkpoint, potential_index[0]) + else: + raise ValueError( + f"{checkpoint} containing more than one `.index.json` file, delete the irrelevant ones." + ) + else: + raise ValueError( + "`checkpoint` should be the path to a file containing a whole state dict, or the index of a sharded " + f"checkpoint, or a folder containing a sharded checkpoint or the whole state dict, but got {checkpoint}." + ) + + if index_filename is not None: + checkpoint_folder = os.path.split(index_filename)[0] + with open(index_filename) as f: + index = json.loads(f.read()) + + if "weight_map" in index: + index = index["weight_map"] + checkpoint_files = sorted(list(set(index.values()))) + checkpoint_files = [os.path.join(checkpoint_folder, f) for f in checkpoint_files] + + # Logic for missing/unexpected keys goes here. + + offload_index = {} + if offload_state_dict: + state_dict_folder = tempfile.mkdtemp() + state_dict_index = {} + + unexpected_keys = set() + model_keys = set(model.state_dict().keys()) + buffer_names = [name for name, _ in model.named_buffers()] + model_devices = {t.device for t in model.state_dict().values() if isinstance(t, torch.Tensor)} + model_physical_devices = model_devices - {torch.device("meta")} + for checkpoint_file in checkpoint_files: + if device_map is None: + # exception for multi-device loading was made for the meta device in torch v2.7.0 + # https://github.com/pytorch/pytorch/blob/v2.6.0/torch/distributed/checkpoint/state_dict.py#L557-L563 + # https://github.com/pytorch/pytorch/blob/v2.7.0-rc2/torch/distributed/checkpoint/state_dict.py#L575-L587 + if is_torch_version(">=", "2.2.0") and ( + (is_torch_version(">=", "2.7.0") and len(model_physical_devices) <= 1) or len(model_devices) <= 1 + ): + from torch.distributed.checkpoint.state_dict import StateDictOptions, set_model_state_dict + + broadcast_from_rank0 &= is_torch_version(">=", "2.4.0") + loaded_checkpoint = ( + load_state_dict(checkpoint_file, device_map=device_map) + if not broadcast_from_rank0 or dist.get_rank() == 0 + else {} + ) + set_model_state_dict( + model, + loaded_checkpoint, + options=StateDictOptions( + full_state_dict=full_state_dict, + strict=strict, + **({"broadcast_from_rank0": broadcast_from_rank0} if is_torch_version(">=", "2.4.0") else {}), + ), + ) + else: + loaded_checkpoint = load_state_dict(checkpoint_file, device_map=device_map) + model.load_state_dict(loaded_checkpoint, strict=strict) + + unexpected_keys.update(set(loaded_checkpoint.keys()) - model_keys) + else: + loaded_checkpoint = load_state_dict(checkpoint_file, device_map=device_map) + + for param_name, param in loaded_checkpoint.items(): + # skip SCB parameter (for 8-bit serialization) + if "SCB" in param_name: + continue + + if param_name not in model_keys: + unexpected_keys.add(param_name) + if not strict: + continue # Skip loading this parameter. + + module_name = param_name + + while len(module_name) > 0 and module_name not in device_map: + module_name = ".".join(module_name.split(".")[:-1]) + if module_name == "" and "" not in device_map: + # TODO: group all errors and raise at the end. + raise ValueError(f"{param_name} doesn't have any device set.") + param_device = device_map[module_name] + new_dtype = dtype + if dtype is not None and torch.is_floating_point(param): + if keep_in_fp32_modules is not None and dtype == torch.float16: + proceed = False + for key in keep_in_fp32_modules: + if ((key in param_name) and (key + "." in param_name)) or key == param_name: + proceed = True + break + if proceed: + new_dtype = torch.float32 + + if "weight" in param_name and param_name.replace("weight", "SCB") in loaded_checkpoint.keys(): + if param.dtype == torch.int8: + fp16_statistics = loaded_checkpoint[param_name.replace("weight", "SCB")] + else: + fp16_statistics = None + + if param_device == "disk": + if offload_buffers or param_name not in buffer_names: + if new_dtype is None: + new_dtype = param.dtype + if offload_8bit_bnb: + quantize_and_offload_8bit( + model, param, param_name, new_dtype, offload_folder, offload_index, fp16_statistics + ) + continue + else: + set_module_tensor_to_device(model, param_name, "meta", dtype=new_dtype) + offload_weight(param, param_name, offload_folder, index=offload_index) + elif param_device == "cpu" and offload_state_dict: + if new_dtype is None: + new_dtype = param.dtype + if offload_8bit_bnb: + quantize_and_offload_8bit( + model, param, param_name, new_dtype, state_dict_folder, state_dict_index, fp16_statistics + ) + else: + set_module_tensor_to_device(model, param_name, "meta", dtype=new_dtype) + offload_weight(param, param_name, state_dict_folder, index=state_dict_index) + else: + set_module_tensor_to_device( + model, + param_name, + param_device, + value=param, + dtype=new_dtype, + fp16_statistics=fp16_statistics, + ) + + # Force Python to clean up. + del loaded_checkpoint + gc.collect() + + if not strict and len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint at {checkpoint} were not used when" + f" initializing {model.__class__.__name__}: {unexpected_keys}. This may or may not be an issue - make sure that the checkpoint does not have unnecessary parameters, or that the model definition correctly corresponds to the checkpoint." + ) + + save_offload_index(offload_index, offload_folder) + + # Load back offloaded state dict on CPU + if offload_state_dict: + load_offloaded_weights(model, state_dict_index, state_dict_folder) + shutil.rmtree(state_dict_folder) + + retie_parameters(model, tied_params) + + +def get_mixed_precision_context_manager(native_amp: bool = False, autocast_kwargs: AutocastKwargs = None): + """ + Return a context manager for autocasting mixed precision + + Args: + native_amp (`bool`, *optional*, defaults to False): + Whether mixed precision is actually enabled. + cache_enabled (`bool`, *optional*, defaults to True): + Whether the weight cache inside autocast should be enabled. + """ + state = AcceleratorState() + if autocast_kwargs is None: + autocast_kwargs = {} + else: + autocast_kwargs = autocast_kwargs.to_kwargs() + if native_amp: + device_type = ( + "cuda" + if (state.distributed_type == DistributedType.XLA and is_torch_xla_available(check_is_gpu=True)) + else state.device.type + ) + if state.mixed_precision == "fp16": + return torch.autocast(device_type=device_type, dtype=torch.float16, **autocast_kwargs) + elif state.mixed_precision in ["bf16", "fp8"] and state.distributed_type in [ + DistributedType.NO, + DistributedType.MULTI_CPU, + DistributedType.MULTI_GPU, + DistributedType.MULTI_MLU, + DistributedType.MULTI_SDAA, + DistributedType.MULTI_MUSA, + DistributedType.MULTI_NPU, + DistributedType.MULTI_XPU, + DistributedType.MULTI_HPU, + DistributedType.MULTI_NEURON, + DistributedType.FSDP, + DistributedType.XLA, + ]: + return torch.autocast(device_type=device_type, dtype=torch.bfloat16, **autocast_kwargs) + else: + return torch.autocast(device_type=device_type, **autocast_kwargs) + else: + return contextlib.nullcontext() + + +def get_grad_scaler(distributed_type: DistributedType = None, **kwargs): + """ + A generic helper which will initialize the correct `GradScaler` implementation based on the environment and return + it. + + Args: + distributed_type (`DistributedType`, *optional*, defaults to None): + The type of distributed environment. + kwargs: + Additional arguments for the utilized `GradScaler` constructor. + """ + if distributed_type == DistributedType.FSDP: + from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler + + return ShardedGradScaler(**kwargs) + if is_torch_xla_available(check_is_gpu=True): + import torch_xla.amp as xamp + + return xamp.GradScaler(**kwargs) + elif is_mlu_available(): + return torch.mlu.amp.GradScaler(**kwargs) + elif is_sdaa_available(): + return torch.sdaa.amp.GradScaler(**kwargs) + elif is_musa_available(): + return torch.musa.amp.GradScaler(**kwargs) + elif is_npu_available(): + return torch.npu.amp.GradScaler(**kwargs) + elif is_hpu_available(): + return torch.amp.GradScaler("hpu", **kwargs) + elif is_xpu_available(): + return torch.amp.GradScaler("xpu", **kwargs) + elif is_mps_available(): + if not is_torch_version(">=", "2.8.0"): + raise ValueError("Grad Scaler with MPS device requires a Pytorch >= 2.8.0") + return torch.amp.GradScaler("mps", **kwargs) + else: + if is_torch_version(">=", "2.3"): + return torch.amp.GradScaler("cuda", **kwargs) + else: + return torch.cuda.amp.GradScaler(**kwargs) + + +def has_offloaded_params(module: torch.nn.Module) -> bool: + """ + Checks if a module has offloaded parameters by checking if the given module has a AlignDevicesHook attached with + offloading enabled + + Args: + module (`torch.nn.Module`): The module to check for an offload hook. + + Returns: + bool: `True` if the module has an offload hook and offloading is enabled, `False` otherwise. + """ + from ..hooks import AlignDevicesHook # avoid circular import + + return hasattr(module, "_hf_hook") and isinstance(module._hf_hook, AlignDevicesHook) and module._hf_hook.offload + + +@contextlib.contextmanager +def align_module_device(module: torch.nn.Module, execution_device: Optional[torch.device] = None): + """ + Context manager that moves a module's parameters to the specified execution device. + + Args: + module (`torch.nn.Module`): + Module with parameters to align. + execution_device (`torch.device`, *optional*): + If provided, overrides the module's execution device within the context. Otherwise, use hook execution + device or pass + """ + if has_offloaded_params(module): + if execution_device is not None: + original_device = module._hf_hook.execution_device + module._hf_hook.execution_device = execution_device + + try: + module._hf_hook.pre_forward(module) + yield + finally: + module._hf_hook.post_forward(module, None) + if execution_device is not None: + module._hf_hook.execution_device = original_device + + elif execution_device is not None: + devices = {name: param.device for name, param in module.named_parameters(recurse=False)} + try: + for name in devices: + set_module_tensor_to_device(module, name, execution_device) + yield + finally: + for name, device in devices.items(): + set_module_tensor_to_device(module, name, device) + + else: + yield diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/offload.py b/kbidu_env/Lib/site-packages/accelerate/utils/offload.py new file mode 100644 index 0000000000000000000000000000000000000000..da8cfaf3ebf7f2965f408e2b952103ac26868647 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/offload.py @@ -0,0 +1,213 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from collections.abc import Mapping +from typing import Optional, Union + +import numpy as np +import torch +from safetensors import safe_open + + +def offload_weight(weight, weight_name, offload_folder, index=None): + dtype = None + # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16. + if str(weight.dtype) == "torch.bfloat16": + # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s. + weight = weight.view(torch.int16) + dtype = "bfloat16" + array = weight.cpu().numpy() + tensor_file = os.path.join(offload_folder, f"{weight_name}.dat") + if index is not None: + if dtype is None: + dtype = str(array.dtype) + index[weight_name] = {"dtype": dtype, "shape": list(array.shape)} + if array.ndim == 0: + array = array[None] + file_array = np.memmap(tensor_file, dtype=array.dtype, mode="w+", shape=array.shape) + file_array[:] = array[:] + file_array.flush() + return index + + +def load_offloaded_weight(weight_file, weight_info): + shape = tuple(weight_info["shape"]) + if shape == (): + # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor + shape = (1,) + + dtype = weight_info["dtype"] + if dtype == "bfloat16": + # NumPy does not support bfloat16 so this was saved as a int16 + dtype = "int16" + + weight = np.memmap(weight_file, dtype=dtype, shape=shape, mode="r") + + if len(weight_info["shape"]) == 0: + weight = weight[0] + weight = torch.tensor(weight) + if weight_info["dtype"] == "bfloat16": + weight = weight.view(torch.bfloat16) + + return weight + + +def save_offload_index(index, offload_folder): + if index is None or len(index) == 0: + # Nothing to save + return + + offload_index_file = os.path.join(offload_folder, "index.json") + if os.path.isfile(offload_index_file): + with open(offload_index_file, encoding="utf-8") as f: + current_index = json.load(f) + else: + current_index = {} + current_index.update(index) + + with open(offload_index_file, "w", encoding="utf-8") as f: + json.dump(current_index, f, indent=2) + + +def offload_state_dict(save_dir: Union[str, os.PathLike], state_dict: dict[str, torch.Tensor]): + """ + Offload a state dict in a given folder. + + Args: + save_dir (`str` or `os.PathLike`): + The directory in which to offload the state dict. + state_dict (`Dict[str, torch.Tensor]`): + The dictionary of tensors to offload. + """ + os.makedirs(save_dir, exist_ok=True) + index = {} + for name, parameter in state_dict.items(): + index = offload_weight(parameter, name, save_dir, index=index) + + # Update index + save_offload_index(index, save_dir) + + +class PrefixedDataset(Mapping): + """ + Will access keys in a given dataset by adding a prefix. + + Args: + dataset (`Mapping`): Any map with string keys. + prefix (`str`): A prefix to add when trying to access any element in the underlying dataset. + """ + + def __init__(self, dataset: Mapping, prefix: str): + self.dataset = dataset + self.prefix = prefix + + def __getitem__(self, key): + return self.dataset[f"{self.prefix}{key}"] + + def __iter__(self): + return iter([key for key in self.dataset if key.startswith(self.prefix)]) + + def __len__(self): + return len(self.dataset) + + +class OffloadedWeightsLoader(Mapping): + """ + A collection that loads weights stored in a given state dict or memory-mapped on disk. + + Args: + state_dict (`Dict[str, torch.Tensor]`, *optional*): + A dictionary parameter name to tensor. + save_folder (`str` or `os.PathLike`, *optional*): + The directory in which the weights are stored (by `offload_state_dict` for instance). + index (`Dict`, *optional*): + A dictionary from weight name to their information (`dtype`/ `shape` or safetensors filename). Will default + to the index saved in `save_folder`. + """ + + def __init__( + self, + state_dict: Optional[dict[str, torch.Tensor]] = None, + save_folder: Optional[Union[str, os.PathLike]] = None, + index: Optional[Mapping] = None, + device=None, + ): + if state_dict is None and save_folder is None and index is None: + raise ValueError("Need either a `state_dict`, a `save_folder` or an `index` containing offloaded weights.") + + self.state_dict = {} if state_dict is None else state_dict + self.save_folder = save_folder + if index is None and save_folder is not None: + with open(os.path.join(save_folder, "index.json")) as f: + index = json.load(f) + self.index = {} if index is None else index + self.all_keys = list(self.state_dict.keys()) + self.all_keys.extend([key for key in self.index if key not in self.all_keys]) + self.device = device + + def __getitem__(self, key: str): + # State dict gets priority + if key in self.state_dict: + return self.state_dict[key] + weight_info = self.index[key] + if weight_info.get("safetensors_file") is not None: + device = "cpu" if self.device is None else self.device + tensor = None + try: + with safe_open(weight_info["safetensors_file"], framework="pt", device=device) as f: + tensor = f.get_tensor(weight_info.get("weight_name", key)) + except TypeError: + # if failed to get_tensor on the device, such as bf16 on mps, try to load it on CPU first + with safe_open(weight_info["safetensors_file"], framework="pt", device="cpu") as f: + tensor = f.get_tensor(weight_info.get("weight_name", key)) + + if "dtype" in weight_info: + tensor = tensor.to(getattr(torch, weight_info["dtype"])) + + if tensor.device != torch.device(device): + tensor = tensor.to(device) + return tensor + + weight_file = os.path.join(self.save_folder, f"{key}.dat") + return load_offloaded_weight(weight_file, weight_info) + + def __iter__(self): + return iter(self.all_keys) + + def __len__(self): + return len(self.all_keys) + + +def extract_submodules_state_dict(state_dict: dict[str, torch.Tensor], submodule_names: list[str]): + """ + Extract the sub state-dict corresponding to a list of given submodules. + + Args: + state_dict (`Dict[str, torch.Tensor]`): The state dict to extract from. + submodule_names (`List[str]`): The list of submodule names we want to extract. + """ + result = {} + for module_name in submodule_names: + # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the + # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance) + result.update( + { + key: param + for key, param in state_dict.items() + if key == module_name or key.startswith(module_name + ".") + } + ) + return result diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/operations.py b/kbidu_env/Lib/site-packages/accelerate/utils/operations.py new file mode 100644 index 0000000000000000000000000000000000000000..869b500f3452f809753586a435c6ea8d910b3f28 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/operations.py @@ -0,0 +1,991 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +A set of basic tensor ops compatible with tpu, gpu, and multigpu +""" + +import pickle +import warnings +from collections.abc import Mapping +from contextlib import contextmanager, nullcontext +from functools import update_wrapper, wraps +from typing import Any + +import torch + +from ..state import AcceleratorState, PartialState +from .constants import TORCH_DISTRIBUTED_OPERATION_TYPES +from .dataclasses import DistributedType, TensorInformation +from .imports import ( + is_npu_available, + is_torch_distributed_available, + is_torch_xla_available, +) +from .versions import is_torch_version + + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + +if is_torch_distributed_available(): + from torch.distributed import ReduceOp + + +def is_torch_tensor(tensor): + return isinstance(tensor, torch.Tensor) + + +def is_torch_xpu_tensor(tensor): + return isinstance( + tensor, + torch.xpu.FloatTensor, + torch.xpu.ByteTensor, + torch.xpu.IntTensor, + torch.xpu.LongTensor, + torch.xpu.HalfTensor, + torch.xpu.DoubleTensor, + torch.xpu.BFloat16Tensor, + ) + + +def is_tensor_information(tensor_info): + return isinstance(tensor_info, TensorInformation) + + +def is_namedtuple(data): + """ + Checks if `data` is a `namedtuple` or not. Can have false positives, but only if a user is trying to mimic a + `namedtuple` perfectly. + """ + return isinstance(data, tuple) and hasattr(data, "_asdict") and hasattr(data, "_fields") + + +def honor_type(obj, generator): + """ + Cast a generator to the same type as obj (list, tuple, or namedtuple) + """ + # Some objects may not be able to instantiate from a generator directly + if is_namedtuple(obj): + return type(obj)(*list(generator)) + else: + return type(obj)(generator) + + +def recursively_apply(func, data, *args, test_type=is_torch_tensor, error_on_other_type=False, **kwargs): + """ + Recursively apply a function on a data structure that is a nested list/tuple/dictionary of a given base type. + + Args: + func (`callable`): + The function to recursively apply. + data (nested list/tuple/dictionary of `main_type`): + The data on which to apply `func` + *args: + Positional arguments that will be passed to `func` when applied on the unpacked data. + main_type (`type`, *optional*, defaults to `torch.Tensor`): + The base type of the objects to which apply `func`. + error_on_other_type (`bool`, *optional*, defaults to `False`): + Whether to return an error or not if after unpacking `data`, we get on an object that is not of type + `main_type`. If `False`, the function will leave objects of types different than `main_type` unchanged. + **kwargs (additional keyword arguments, *optional*): + Keyword arguments that will be passed to `func` when applied on the unpacked data. + + Returns: + The same data structure as `data` with `func` applied to every object of type `main_type`. + """ + if isinstance(data, (tuple, list)): + return honor_type( + data, + ( + recursively_apply( + func, o, *args, test_type=test_type, error_on_other_type=error_on_other_type, **kwargs + ) + for o in data + ), + ) + elif isinstance(data, Mapping): + return type(data)( + { + k: recursively_apply( + func, v, *args, test_type=test_type, error_on_other_type=error_on_other_type, **kwargs + ) + for k, v in data.items() + } + ) + elif test_type(data): + return func(data, *args, **kwargs) + elif error_on_other_type: + raise TypeError( + f"Unsupported types ({type(data)}) passed to `{func.__name__}`. Only nested list/tuple/dicts of " + f"objects that are valid for `{test_type.__name__}` should be passed." + ) + return data + + +def send_to_device(tensor, device, non_blocking=False, skip_keys=None): + """ + Recursively sends the elements in a nested list/tuple/dictionary of tensors to a given device. + + Args: + tensor (nested list/tuple/dictionary of `torch.Tensor`): + The data to send to a given device. + device (`torch.device`): + The device to send the data to. + non_blocking (`bool`, *optional*, defaults to `False`): + If `True`, the transfer to the device is performed asynchronously, which can overlap + data movement with computation. Only effective when the device supports it (e.g. CUDA). + skip_keys (`str` or `List[str]`, *optional*): + A key or list of keys in a dictionary `tensor` whose values should not be sent to + the given `device`. Entries with these keys are left on their original device. + + Returns: + The same data structure as `tensor` with all tensors sent to the proper device. + """ + if is_torch_tensor(tensor) or hasattr(tensor, "to"): + # `torch.Tensor.to("npu")` could not find context when called for the first time (see this [issue](https://gitee.com/ascend/pytorch/issues/I8KECW?from=project-issue)). + if device == "npu": + device = "npu:0" + try: + return tensor.to(device, non_blocking=non_blocking) + except TypeError: # .to() doesn't accept non_blocking as kwarg + return tensor.to(device) + except AssertionError as error: + # `torch.Tensor.to()` is not supported by `torch_npu` (see this [issue](https://github.com/Ascend/pytorch/issues/16)). + # This call is inside the try-block since is_npu_available is not supported by torch.compile. + if is_npu_available(): + if isinstance(device, int): + device = f"npu:{device}" + else: + raise error + try: + return tensor.to(device, non_blocking=non_blocking) + except TypeError: # .to() doesn't accept non_blocking as kwarg + return tensor.to(device) + elif isinstance(tensor, (tuple, list)): + return honor_type( + tensor, (send_to_device(t, device, non_blocking=non_blocking, skip_keys=skip_keys) for t in tensor) + ) + elif isinstance(tensor, Mapping): + if isinstance(skip_keys, str): + skip_keys = [skip_keys] + elif skip_keys is None: + skip_keys = [] + return type(tensor)( + { + k: t if k in skip_keys else send_to_device(t, device, non_blocking=non_blocking, skip_keys=skip_keys) + for k, t in tensor.items() + } + ) + else: + return tensor + + +def get_data_structure(data): + """ + Recursively gathers the information needed to rebuild a nested list/tuple/dictionary of tensors. + + Args: + data (nested list/tuple/dictionary of `torch.Tensor`): + The data to send to analyze. + + Returns: + The same data structure as `data` with [`~utils.TensorInformation`] instead of tensors. + """ + + def _get_data_structure(tensor): + return TensorInformation(shape=tensor.shape, dtype=tensor.dtype) + + return recursively_apply(_get_data_structure, data) + + +def get_shape(data): + """ + Recursively gathers the shape of a nested list/tuple/dictionary of tensors as a list. + + Args: + data (nested list/tuple/dictionary of `torch.Tensor`): + The data to send to analyze. + + Returns: + The same data structure as `data` with lists of tensor shapes instead of tensors. + """ + + def _get_shape(tensor): + return list(tensor.shape) + + return recursively_apply(_get_shape, data) + + +def initialize_tensors(data_structure): + """ + Recursively initializes tensors from a nested list/tuple/dictionary of [`~utils.TensorInformation`]. + + Returns: + The same data structure as `data` with tensors instead of [`~utils.TensorInformation`]. + """ + + def _initialize_tensor(tensor_info): + return torch.empty(*tensor_info.shape, dtype=tensor_info.dtype) + + return recursively_apply(_initialize_tensor, data_structure, test_type=is_tensor_information) + + +def find_batch_size(data): + """ + Recursively finds the batch size in a nested list/tuple/dictionary of lists of tensors. + + Args: + data (nested list/tuple/dictionary of `torch.Tensor`): The data from which to find the batch size. + + Returns: + `int`: The batch size. + """ + if isinstance(data, (tuple, list, Mapping)) and (len(data) == 0): + raise ValueError(f"Cannot find the batch size from empty {type(data)}.") + + if isinstance(data, (tuple, list)): + return find_batch_size(data[0]) + elif isinstance(data, Mapping): + for k in data.keys(): + return find_batch_size(data[k]) + elif not isinstance(data, torch.Tensor): + raise TypeError(f"Can only find the batch size of tensors but got {type(data)}.") + return data.shape[0] + + +def ignorant_find_batch_size(data): + """ + Same as [`utils.operations.find_batch_size`] except will ignore if `ValueError` and `TypeErrors` are raised + + Args: + data (nested list/tuple/dictionary of `torch.Tensor`): The data from which to find the batch size. + + Returns: + `int`: The batch size. + """ + try: + return find_batch_size(data) + except (ValueError, TypeError): + pass + return None + + +def listify(data): + """ + Recursively finds tensors in a nested list/tuple/dictionary and converts them to a list of numbers. + + Args: + data (nested list/tuple/dictionary of `torch.Tensor`): The data from which to convert to regular numbers. + + Returns: + The same data structure as `data` with lists of numbers instead of `torch.Tensor`. + """ + + def _convert_to_list(tensor): + tensor = tensor.detach().cpu() + if tensor.dtype == torch.bfloat16: + # As of Numpy 1.21.4, NumPy does not support bfloat16 (see + # https://github.com/numpy/numpy/blob/a47ecdea856986cd60eabbd53265c2ca5916ad5d/doc/source/user/basics.types.rst ). + # Until Numpy adds bfloat16, we must convert float32. + tensor = tensor.to(torch.float32) + return tensor.tolist() + + return recursively_apply(_convert_to_list, data) + + +def _tpu_gather(tensor): + def _tpu_gather_one(tensor): + if tensor.ndim == 0: + tensor = tensor.clone()[None] + + # Can only gather contiguous tensors + if not tensor.is_contiguous(): + tensor = tensor.contiguous() + return xm.all_gather(tensor) + + res = recursively_apply(_tpu_gather_one, tensor, error_on_other_type=True) + xm.mark_step() + return res + + +def _gpu_gather(tensor): + state = PartialState() + gather_op = torch.distributed.all_gather_into_tensor + + # NOTE: need manually synchronize to workaourd a INT64 collectives bug in oneCCL before torch 2.9.0 + if state.device.type == "xpu" and is_torch_version("<=", "2.8"): + torch.xpu.synchronize() + + def _gpu_gather_one(tensor): + if tensor.ndim == 0: + tensor = tensor.clone()[None] + + # Can only gather contiguous tensors + if not tensor.is_contiguous(): + tensor = tensor.contiguous() + + if state.backend is not None and state.backend != "gloo": + # We use `empty` as `all_gather_into_tensor` slightly + # differs from `all_gather` for better efficiency, + # and we rely on the number of items in the tensor + # rather than its direct shape + output_tensors = torch.empty( + state.num_processes * tensor.numel(), + dtype=tensor.dtype, + device=state.device, + ) + gather_op(output_tensors, tensor) + return output_tensors.view(-1, *tensor.size()[1:]) + else: + # a backend of `None` is always CPU + # also gloo does not support `all_gather_into_tensor`, + # which will result in a larger memory overhead for the op + output_tensors = [torch.empty_like(tensor) for _ in range(state.num_processes)] + torch.distributed.all_gather(output_tensors, tensor) + return torch.cat(output_tensors, dim=0) + + return recursively_apply(_gpu_gather_one, tensor, error_on_other_type=True) + + +class DistributedOperationException(Exception): + """ + An exception class for distributed operations. Raised if the operation cannot be performed due to the shape of the + tensors. + """ + + pass + + +def verify_operation(function): + """ + Verifies that `tensor` is the same shape across all processes. Only ran if `PartialState().debug` is `True`. + """ + + @wraps(function) + def wrapper(*args, **kwargs): + if PartialState().distributed_type == DistributedType.NO or not PartialState().debug: + return function(*args, **kwargs) + operation = f"{function.__module__}.{function.__name__}" + if "tensor" in kwargs: + tensor = kwargs["tensor"] + else: + tensor = args[0] + if PartialState().device.type != find_device(tensor).type: + raise DistributedOperationException( + f"One or more of the tensors passed to {operation} were not on the {tensor.device.type} while the `Accelerator` is configured for {PartialState().device.type}. " + f"Please move it to the {PartialState().device.type} before calling {operation}." + ) + shapes = get_shape(tensor) + output = gather_object([shapes]) + if output[0] is not None: + are_same = output.count(output[0]) == len(output) + if not are_same: + process_shape_str = "\n - ".join([f"Process {i}: {shape}" for i, shape in enumerate(output)]) + raise DistributedOperationException( + f"Cannot apply desired operation due to shape mismatches. " + "All shapes across devices must be valid." + f"\n\nOperation: `{operation}`\nInput shapes:\n - {process_shape_str}" + ) + return function(*args, **kwargs) + + return wrapper + + +def chained_operation(function): + """ + Checks that `verify_operation` failed and if so reports a more helpful error chaining the existing + `DistributedOperationException`. + """ + + @wraps(function) + def wrapper(*args, **kwargs): + try: + return function(*args, **kwargs) + except DistributedOperationException as e: + operation = f"{function.__module__}.{function.__name__}" + raise DistributedOperationException( + f"Error found while calling `{operation}`. Please see the earlier error for more details." + ) from e + + return wrapper + + +@verify_operation +def gather(tensor): + """ + Recursively gather tensor in a nested list/tuple/dictionary of tensors from all devices. + + Args: + tensor (nested list/tuple/dictionary of `torch.Tensor`): + The data to gather. + + Returns: + The same data structure as `tensor` with all tensors sent to the proper device. + """ + if PartialState().distributed_type == DistributedType.XLA: + return _tpu_gather(tensor) + elif PartialState().distributed_type in TORCH_DISTRIBUTED_OPERATION_TYPES: + return _gpu_gather(tensor) + else: + return tensor + + +def _neuron_gather_object(object: Any): + """Gather picklable objects from all ranks with padded allgather sizes. + + On Neuron devices every unique tensor shape triggers a new NEFF compilation. The standard + ``all_gather_object`` sizes its byte tensor to the exact max pickle size, which varies per call + and causes unbounded compilation cache growth. This variant rounds the allgather size up to + the next power of 2 so the number of distinct compiled shapes stays bounded to O(log(max_size)). + """ + import io + + state = PartialState() + device = state.device + group_size = state.num_processes + + # Serialize + buf = io.BytesIO() + pickle.dump(object, buf) + raw_bytes = buf.getvalue() + local_size = len(raw_bytes) + + # Exchange sizes – each rank sends a single int64 + local_size_tensor = torch.tensor([local_size], dtype=torch.long, device=device) + size_list = [torch.zeros(1, dtype=torch.long, device=device) for _ in range(group_size)] + torch.distributed.all_gather(size_list, local_size_tensor) + max_size = int(max(s.item() for s in size_list)) + + # Round up to next power of 2 (starting at 1MB) so the allgather shape stays stable — O(log(max_size)) distinct shapes + padded_size = 1024 * 1024 + while padded_size < max_size: + padded_size <<= 1 + + # Pack local bytes into a uint8 tensor of the padded size (zero-pads beyond local_size) + byte_storage = torch.ByteStorage._from_buffer(raw_bytes) + input_tensor = torch.ByteTensor(byte_storage).to(device) + input_tensor.resize_(padded_size) + + # Allgather – shape is (padded_size,) on every rank, stable across steps + coalesced = torch.empty(padded_size * group_size, dtype=torch.uint8, device=device) + output_tensors = [coalesced[padded_size * i : padded_size * (i + 1)] for i in range(group_size)] + torch.distributed.all_gather(output_tensors, input_tensor) + + # Deserialize and flatten (each rank's object is a list, flatten like _gpu_gather_object) + result = [] + for i, tensor in enumerate(output_tensors): + obj_size = int(size_list[i].item()) + obj_bytes = tensor[:obj_size].cpu().numpy().tobytes() + obj = pickle.loads(obj_bytes) + if isinstance(obj, list): + result.extend(obj) + else: + result.append(obj) + return result + + +def _gpu_gather_object(object: Any): + output_objects = [None for _ in range(PartialState().num_processes)] + torch.distributed.all_gather_object(output_objects, object) + # all_gather_object returns a list of lists, so we need to flatten it + return [x for y in output_objects for x in y] + + +def gather_object(object: Any): + """ + Recursively gather object in a nested list/tuple/dictionary of objects from all devices. + + Args: + object (nested list/tuple/dictionary of picklable object): + The data to gather. + + Returns: + The same data structure as `object` with all the objects sent to every device. + """ + if PartialState().distributed_type == DistributedType.XLA: + raise NotImplementedError("gather objects in TPU is not supported") + elif PartialState().distributed_type in TORCH_DISTRIBUTED_OPERATION_TYPES: + if PartialState().device.type == "neuron": + return _neuron_gather_object(object) + return _gpu_gather_object(object) + else: + return object + + +def _gpu_broadcast(data, src=0): + def _gpu_broadcast_one(tensor, src=0): + torch.distributed.broadcast(tensor, src=src) + return tensor + + return recursively_apply(_gpu_broadcast_one, data, error_on_other_type=True, src=src) + + +def _tpu_broadcast(tensor, src=0, name="broadcast tensor"): + if isinstance(tensor, (list, tuple)): + return honor_type(tensor, (_tpu_broadcast(t, name=f"{name}_{i}") for i, t in enumerate(tensor))) + elif isinstance(tensor, Mapping): + return type(tensor)({k: _tpu_broadcast(v, name=f"{name}_{k}") for k, v in tensor.items()}) + return xm.mesh_reduce(name, tensor, lambda x: x[src]) + + +TENSOR_TYPE_TO_INT = { + torch.float: 1, + torch.double: 2, + torch.half: 3, + torch.bfloat16: 4, + torch.uint8: 5, + torch.int8: 6, + torch.int16: 7, + torch.int32: 8, + torch.int64: 9, + torch.bool: 10, +} + +TENSOR_INT_TO_DTYPE = {v: k for k, v in TENSOR_TYPE_TO_INT.items()} + + +def gather_tensor_shape(tensor): + """ + Grabs the shape of `tensor` only available on one process and returns a tensor of its shape + """ + # Allocate 80 bytes to store the shape + max_tensor_dimension = 2**20 + state = PartialState() + base_tensor = torch.empty(max_tensor_dimension, dtype=torch.int, device=state.device) + + # Since PyTorch can't just send a tensor to another GPU without + # knowing its size, we store the size of the tensor with data + # in an allocation + if tensor is not None: + shape = tensor.shape + tensor_dtype = TENSOR_TYPE_TO_INT[tensor.dtype] + base_tensor[: len(shape) + 1] = torch.tensor(list(shape) + [tensor_dtype], dtype=int) + # Perform a reduction to copy the size data onto all GPUs + base_tensor = reduce(base_tensor, reduction="sum") + base_tensor = base_tensor[base_tensor.nonzero()] + # The last non-zero data contains the coded dtype the source tensor is + dtype = int(base_tensor[-1:][0]) + base_tensor = base_tensor[:-1] + return base_tensor, dtype + + +def copy_tensor_to_devices(tensor=None) -> torch.Tensor: + """ + Copies a tensor that only exists on a single device and broadcasts it to other devices. Differs from `broadcast` as + each worker doesn't need to know its shape when used (and tensor can be `None`) + + Args: + tensor (`torch.tensor`): + The tensor that should be sent to all devices. Must only have it be defined on a single device, the rest + should be `None`. + """ + state = PartialState() + shape, dtype = gather_tensor_shape(tensor) + if tensor is None: + tensor = torch.zeros(shape, dtype=TENSOR_INT_TO_DTYPE[dtype]).to(state.device) + return reduce(tensor, reduction="sum") + + +@verify_operation +def broadcast(tensor, from_process: int = 0): + """ + Recursively broadcast tensor in a nested list/tuple/dictionary of tensors to all devices. + + Args: + tensor (nested list/tuple/dictionary of `torch.Tensor`): + The data to gather. + from_process (`int`, *optional*, defaults to 0): + The process from which to send the data + + Returns: + The same data structure as `tensor` with all tensors broadcasted to the proper device. + """ + if PartialState().distributed_type == DistributedType.XLA: + return _tpu_broadcast(tensor, src=from_process, name="accelerate.utils.broadcast") + elif PartialState().distributed_type in TORCH_DISTRIBUTED_OPERATION_TYPES: + return _gpu_broadcast(tensor, src=from_process) + else: + return tensor + + +def _neuron_broadcast_object_list(object_list, from_process: int = 0): + """Broadcast a list of picklable objects with padded tensor sizes. + + On Neuron devices ``ProcessGroupNeuron.broadcast()`` is implemented as an allreduce, + so every unique tensor shape triggers a new NEFF compilation. The standard + ``broadcast_object_list`` sizes its byte tensor to the exact serialized length, which + varies per call and causes unbounded compilation cache growth. This variant rounds the + broadcast tensor size up to the next power of 2, bounding the number of distinct + compiled shapes to ~O(log(max_size)). + """ + import io + + state = PartialState() + device = state.device + + # --- 1. Serialize on the source rank, get sizes --------------------------- + if state.process_index == from_process: + buf = io.BytesIO() + pickle.dump(object_list, buf) + raw_bytes = buf.getvalue() + local_size = len(raw_bytes) + else: + raw_bytes = b"" + local_size = 0 + + # Broadcast the serialized size from root so every rank knows the padded length. + # Use allreduce(SUM) directly: root has the real size, others have 0. + size_tensor = torch.tensor([local_size], dtype=torch.long, device=device) + torch.distributed.all_reduce(size_tensor) + real_size = int(size_tensor.item()) + + # Round up to next power of 2 (starting at 1MB) to keep the number of distinct shapes O(log(max_size)) + padded_size = 1024 * 1024 + while padded_size < real_size: + padded_size <<= 1 + + # --- 2. Build the byte tensor and broadcast -------------------------------- + if state.process_index == from_process: + byte_storage = torch.ByteStorage._from_buffer(raw_bytes) + data_tensor = torch.ByteTensor(byte_storage).to(device) + data_tensor.resize_(padded_size) + else: + data_tensor = torch.zeros(padded_size, dtype=torch.uint8, device=device) + + torch.distributed.broadcast(data_tensor, src=from_process) + + # --- 3. Deserialize on all ranks ------------------------------------------ + result = pickle.loads(data_tensor[:real_size].cpu().numpy().tobytes()) + for i in range(len(object_list)): + object_list[i] = result[i] + return object_list + + +def broadcast_object_list(object_list, from_process: int = 0): + """ + Broadcast a list of picklable objects from one process to the others. + + Args: + object_list (list of picklable objects): + The list of objects to broadcast. This list will be modified inplace. + from_process (`int`, *optional*, defaults to 0): + The process from which to send the data. + + Returns: + The same list containing the objects from process 0. + """ + if PartialState().distributed_type == DistributedType.XLA: + for i, obj in enumerate(object_list): + object_list[i] = xm.mesh_reduce("accelerate.utils.broadcast_object_list", obj, lambda x: x[from_process]) + elif PartialState().distributed_type in TORCH_DISTRIBUTED_OPERATION_TYPES: + if PartialState().device.type == "neuron": + _neuron_broadcast_object_list(object_list, from_process=from_process) + else: + torch.distributed.broadcast_object_list(object_list, src=from_process) + return object_list + + +def slice_tensors(data, tensor_slice, process_index=None, num_processes=None): + """ + Recursively takes a slice in a nested list/tuple/dictionary of tensors. + + Args: + data (nested list/tuple/dictionary of `torch.Tensor`): + The data to slice. + tensor_slice (`slice`): + The slice to take. + + Returns: + The same data structure as `data` with all the tensors slices. + """ + + def _slice_tensor(tensor, tensor_slice): + return tensor[tensor_slice] + + return recursively_apply(_slice_tensor, data, tensor_slice) + + +def concatenate(data, dim=0): + """ + Recursively concatenate the tensors in a nested list/tuple/dictionary of lists of tensors with the same shape. + If there is only a single batch of data, it is returned as-is. + + Args: + data (nested list/tuple/dictionary of lists of tensors `torch.Tensor`): + The data to concatenate. + dim (`int`, *optional*, defaults to 0): + The dimension on which to concatenate. + + Returns: + The same data structure as `data` with all the tensors concatenated. + """ + if isinstance(data[0], (tuple, list)): + return honor_type(data[0], (concatenate([d[i] for d in data], dim=dim) for i in range(len(data[0])))) + elif isinstance(data[0], Mapping): + return type(data[0])({k: concatenate([d[k] for d in data], dim=dim) for k in data[0].keys()}) + elif isinstance(data[0], torch.Tensor): + return torch.cat(data, dim=dim) + elif isinstance(data, (tuple, list)) and len(data) == 1: + return data[0] + else: + raise TypeError(f"Can only concatenate tensors but got {type(data[0])}") + + +class CannotPadNestedTensorWarning(UserWarning): + pass + + +@chained_operation +def pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False): + """ + Recursively pad the tensors in a nested list/tuple/dictionary of tensors from all devices to the same size so they + can safely be gathered. + + Args: + tensor (nested list/tuple/dictionary of `torch.Tensor`): + The data to gather. + dim (`int`, *optional*, defaults to 0): + The dimension on which to pad. + pad_index (`int`, *optional*, defaults to 0): + The value with which to pad. + pad_first (`bool`, *optional*, defaults to `False`): + Whether to pad at the beginning or the end. + """ + + def _pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False): + if getattr(tensor, "is_nested", False): + warnings.warn( + "Cannot pad nested tensors without more information. Leaving unprocessed.", + CannotPadNestedTensorWarning, + ) + return tensor + if dim >= len(tensor.shape) or dim < -len(tensor.shape): + return tensor + # Convert negative dimensions to non-negative + if dim < 0: + dim += len(tensor.shape) + + # Gather all sizes + size = torch.tensor(tensor.shape, device=tensor.device)[None] + sizes = gather(size).cpu() + # Then pad to the maximum size + max_size = max(s[dim] for s in sizes) + if max_size == tensor.shape[dim]: + return tensor + + old_size = tensor.shape + new_size = list(old_size) + new_size[dim] = max_size + new_tensor = tensor.new_zeros(tuple(new_size)) + pad_index + if pad_first: + indices = tuple( + slice(max_size - old_size[dim], max_size) if i == dim else slice(None) for i in range(len(new_size)) + ) + else: + indices = tuple(slice(0, old_size[dim]) if i == dim else slice(None) for i in range(len(new_size))) + new_tensor[indices] = tensor + return new_tensor + + return recursively_apply( + _pad_across_processes, tensor, error_on_other_type=True, dim=dim, pad_index=pad_index, pad_first=pad_first + ) + + +def pad_input_tensors(tensor, batch_size, num_processes, dim=0): + """ + Takes a `tensor` of arbitrary size and pads it so that it can work given `num_processes` needed dimensions. + + New tensors are just the last input repeated. + + E.g.: + Tensor: ([3,4,4]) Num processes: 4 Expected result shape: ([4,4,4]) + + """ + + def _pad_input_tensors(tensor, batch_size, num_processes, dim=0): + remainder = batch_size // num_processes + last_inputs = batch_size - (remainder * num_processes) + if batch_size // num_processes == 0: + to_pad = num_processes - batch_size + else: + to_pad = num_processes - (batch_size // num_processes) + # In the rare case that `to_pad` is negative, + # we need to pad the last inputs - the found `to_pad` + if last_inputs > to_pad & to_pad < 1: + to_pad = last_inputs - to_pad + old_size = tensor.shape + new_size = list(old_size) + new_size[0] = batch_size + to_pad + new_tensor = tensor.new_zeros(tuple(new_size)) + indices = tuple(slice(0, old_size[dim]) if i == dim else slice(None) for i in range(len(new_size))) + new_tensor[indices] = tensor + return new_tensor + + return recursively_apply( + _pad_input_tensors, + tensor, + error_on_other_type=True, + batch_size=batch_size, + num_processes=num_processes, + dim=dim, + ) + + +@verify_operation +def reduce(tensor, reduction="mean", scale=1.0): + """ + Recursively reduce the tensors in a nested list/tuple/dictionary of lists of tensors across all processes by the + mean of a given operation. + + Args: + tensor (nested list/tuple/dictionary of `torch.Tensor`): + The data to reduce. + reduction (`str`, *optional*, defaults to `"mean"`): + A reduction method. Can be of "mean", "sum", "max", or "none" + scale (`float`, *optional*): + A default scaling value to be applied after the reduce, only valid on XLA. + + Returns: + The same data structure as `data` with all the tensors reduced. + """ + + def _reduce_across_processes(tensor, reduction="mean", scale=1.0): + state = PartialState() + cloned_tensor = tensor.clone() + if state.distributed_type == DistributedType.NO: + return cloned_tensor + if state.distributed_type == DistributedType.XLA: + # Some processes may have different HLO graphs than other + # processes, for example in the breakpoint API + # accelerator.set_trigger(). Use mark_step to make HLOs + # the same on all processes. + xm.mark_step() + xla_op = xm.REDUCE_MAX if reduction == "max" else xm.REDUCE_SUM + xm.all_reduce(xla_op, [cloned_tensor], scale) + xm.mark_step() + elif state.distributed_type.value in TORCH_DISTRIBUTED_OPERATION_TYPES: + torch_op = ReduceOp.MAX if reduction == "max" else ReduceOp.SUM + torch.distributed.all_reduce(cloned_tensor, torch_op) + if reduction == "mean": + cloned_tensor /= state.num_processes + return cloned_tensor + + return recursively_apply( + _reduce_across_processes, tensor, error_on_other_type=True, reduction=reduction, scale=scale + ) + + +def convert_to_fp32(tensor): + """ + Recursively converts the elements nested list/tuple/dictionary of tensors in FP16/BF16 precision to FP32. + + Args: + tensor (nested list/tuple/dictionary of `torch.Tensor`): + The data to convert from FP16/BF16 to FP32. + + Returns: + The same data structure as `tensor` with all tensors that were in FP16/BF16 precision converted to FP32. + """ + + def _convert_to_fp32(tensor): + return tensor.float() + + def _is_fp16_bf16_tensor(tensor): + return (is_torch_tensor(tensor) or hasattr(tensor, "dtype")) and tensor.dtype in ( + torch.float16, + torch.bfloat16, + ) + + return recursively_apply(_convert_to_fp32, tensor, test_type=_is_fp16_bf16_tensor) + + +class ConvertOutputsToFp32: + """ + Decorator to apply to a function outputting tensors (like a model forward pass) that ensures the outputs in FP16 + precision will be convert back to FP32. + + Args: + model_forward (`Callable`): + The function which outputs we want to treat. + + Returns: + The same function as `model_forward` but with converted outputs. + """ + + def __init__(self, model_forward): + self.model_forward = model_forward + update_wrapper(self, model_forward) + + def __call__(self, *args, **kwargs): + return convert_to_fp32(self.model_forward(*args, **kwargs)) + + def __getstate__(self): + raise pickle.PicklingError( + "Cannot pickle a prepared model with automatic mixed precision, please unwrap the model with `Accelerator.unwrap_model(model)` before pickling it." + ) + + +def convert_outputs_to_fp32(model_forward): + model_forward = ConvertOutputsToFp32(model_forward) + + def forward(*args, **kwargs): + return model_forward(*args, **kwargs) + + # To act like a decorator so that it can be popped when doing `extract_model_from_parallel` + forward.__wrapped__ = model_forward + + return forward + + +def find_device(data): + """ + Finds the device on which a nested dict/list/tuple of tensors lies (assuming they are all on the same device). + + Args: + (nested list/tuple/dictionary of `torch.Tensor`): The data we want to know the device of. + """ + if isinstance(data, Mapping): + for obj in data.values(): + device = find_device(obj) + if device is not None: + return device + elif isinstance(data, (tuple, list)): + for obj in data: + device = find_device(obj) + if device is not None: + return device + elif isinstance(data, torch.Tensor): + return data.device + + +@contextmanager +def GatheredParameters(params, modifier_rank=None, fwd_module=None, enabled=True): + """ + Wrapper around `deepspeed.runtime.zero.GatheredParameters`, but if Zero-3 is not enabled, will be a no-op context + manager. + """ + # We need to use the `AcceleratorState` here since it has access to the deepspeed plugin + if AcceleratorState().distributed_type != DistributedType.DEEPSPEED or ( + AcceleratorState().deepspeed_plugin is not None + and not AcceleratorState().deepspeed_plugin.is_zero3_init_enabled() + ): + gather_param_context = nullcontext() + else: + import deepspeed + + gather_param_context = deepspeed.zero.GatheredParameters( + params, modifier_rank=modifier_rank, fwd_module=fwd_module, enabled=enabled + ) + with gather_param_context: + yield diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/other.py b/kbidu_env/Lib/site-packages/accelerate/utils/other.py new file mode 100644 index 0000000000000000000000000000000000000000..1f7c1738126d63bfbd1ff988a9dd162c66434097 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/other.py @@ -0,0 +1,594 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import platform +import re +import socket +from codecs import encode +from collections import OrderedDict +from functools import partial, reduce +from types import MethodType +from typing import Optional + +import numpy as np +import torch +from packaging.version import Version +from safetensors.torch import save_file as safe_save_file + +from ..commands.config.default import write_basic_config # noqa: F401 +from ..logging import get_logger +from ..state import PartialState +from .constants import FSDP_PYTORCH_VERSION +from .dataclasses import DistributedType +from .imports import ( + is_deepspeed_available, + is_numpy_available, + is_torch_distributed_available, + is_torch_xla_available, + is_weights_only_available, +) +from .modeling import id_tensor_storage +from .transformer_engine import convert_model +from .versions import is_torch_version + + +logger = get_logger(__name__) + + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + +def is_compiled_module(module: torch.nn.Module) -> bool: + """ + Check whether the module was compiled with torch.compile() + """ + if not hasattr(torch, "_dynamo"): + return False + + return isinstance(module, torch._dynamo.eval_frame.OptimizedModule) + + +def has_compiled_regions(module: torch.nn.Module) -> bool: + """ + Check whether the module has submodules that were compiled with `torch.compile()`. + """ + if not hasattr(torch, "_dynamo"): + return False + + if module._modules: + for submodule in module.modules(): + if isinstance(submodule, torch._dynamo.eval_frame.OptimizedModule): + return True + + return False + + +def is_repeated_blocks(module: torch.nn.Module) -> bool: + """ + Check whether the module is a repeated block, i.e. `torch.nn.ModuleList` with all children of the same class. This + is useful to determine whether we should apply regional compilation to the module. + """ + + return ( + isinstance(module, torch.nn.ModuleList) + and len(module) > 0 + and all(isinstance(m, module[0].__class__) for m in module) + ) + + +def has_repeated_blocks(module: torch.nn.Module) -> bool: + """ + Check whether the module has repeated blocks, i.e. `torch.nn.ModuleList` with all children of the same class, at + any level of the module hierarchy. This is useful to determine whether we should apply regional compilation to the + module. + """ + if module._modules: + for submodule in module.modules(): + if is_repeated_blocks(submodule): + return True + + return False + + +def compile_regions(module: torch.nn.Module, **compile_kwargs) -> torch.nn.Module: + """ + Performs regional compilation where we target repeated blocks of the same class and compile them sequentially to + hit the compiler's cache. For example, in `GPT2LMHeadModel`, the repeated block/class is `GPT2Block`, and can be + accessed as `model.transformer.h[0]`. The rest of the model (e.g. model.lm_head) is compiled separately. + + This allows us to speed up the compilation overhead / cold start of models like LLMs and Transformers in general. + See https://pytorch.org/tutorials/recipes/regional_compilation.html for more details. + + Args: + module (`torch.nn.Module`): + The model to compile. + **compile_kwargs: + Additional keyword arguments to pass to `torch.compile()`. + + Returns: + `torch.nn.Module`: A new instance of the model with some compiled regions. + + Example: + ```python + >>> from accelerate.utils import compile_regions + >>> from transformers import AutoModelForCausalLM + + >>> model = AutoModelForCausalLM.from_pretrained("gpt2") + >>> compiled_model = compile_regions(model, mode="reduce-overhead") + >>> compiled_model.transformer.h[0] + OptimizedModule( + (_orig_mod): GPT2Block( + (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True) + (attn): GPT2Attention( + (c_attn): Conv1D(nf=2304, nx=768) + (c_proj): Conv1D(nf=768, nx=768) + (attn_dropout): Dropout(p=0.1, inplace=False) + (resid_dropout): Dropout(p=0.1, inplace=False) + ) + (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True) + (mlp): GPT2MLP( + (c_fc): Conv1D(nf=3072, nx=768) + (c_proj): Conv1D(nf=768, nx=3072) + (act): NewGELUActivation() + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + ) + ``` + """ + + def _compile_regions(module: torch.nn.Module, **compile_kwargs) -> torch.nn.Module: + if is_repeated_blocks(module): + new_module = torch.nn.ModuleList() + for submodule in module: + new_module.append(torch.compile(submodule, **compile_kwargs)) + elif has_repeated_blocks(module): + new_module = module.__class__.__new__(module.__class__) + new_module.__dict__.update(module.__dict__) + new_module._modules = {} + for name, submodule in module.named_children(): + new_module.add_module(name, _compile_regions(submodule, **compile_kwargs)) + else: + new_module = torch.compile(module, **compile_kwargs) + + return new_module + + new_module = _compile_regions(module, **compile_kwargs) + + if "_orig_mod" not in new_module.__dict__: + # Keeps a reference to the original module to decompile/unwrap it later + new_module.__dict__["_orig_mod"] = module + + return new_module + + +def compile_regions_deepspeed(module: torch.nn.Module, **compile_kwargs): + """ + Performs regional compilation the same way as `compile_regions`, but specifically for `DeepSpeedEngine.module`. + Since the model is wrapped in a `DeepSpeedEngine` and has many added hooks, offloaded parameters, etc that + `torch.compile(...)` interferes with, version of trgional compilation uses the inplace `module.compile()` method + instead. + + Args: + module (`torch.nn.Module`): + The model to compile. + **compile_kwargs: + Additional keyword arguments to pass to `module.compile()`. + """ + + if is_repeated_blocks(module): + for submodule in module: + submodule.compile(**compile_kwargs) + elif has_repeated_blocks(module): + for child in module.children(): + compile_regions_deepspeed(child, **compile_kwargs) + else: # leaf node + module.compile(**compile_kwargs) + + +def compile_regions_fsdp2(module: torch.nn.Module, **compile_kwargs) -> torch.nn.Module: + """ + Like `compile_regions`, but uses the in-place `module.compile()` instead of `torch.compile(module)`. + + Needed for the FSDP2 prepare path: `torch.compile(module)` returns an `OptimizedModule` whose `__call__` + bypasses `nn.Module._call_impl`, so forward/pre hooks added later by `fully_shard` never fire and per-layer + all-gather/reshard is lost. The in-place `module.compile()` keeps `_call_impl` (and its runtime hook check) + on the call path, so FSDP hooks installed afterwards still run. + + Args: + module (`torch.nn.Module`): + The model to compile. + **compile_kwargs: + Additional keyword arguments to pass to `module.compile()`. + """ + if is_repeated_blocks(module): + for submodule in module: + submodule.compile(**compile_kwargs) + elif has_repeated_blocks(module): + for child in module.children(): + compile_regions_fsdp2(child, **compile_kwargs) + else: # leaf node + module.compile(**compile_kwargs) + return module + + +def model_has_dtensor(model: torch.nn.Module) -> bool: + """ + Check if the model has DTensor parameters. + + Args: + model (`torch.nn.Module`): + The model to check. + + Returns: + `bool`: Whether the model has DTensor parameters. + """ + if is_torch_version(">=", "2.5.0"): + from torch.distributed.tensor import DTensor + else: + # from torch 2.0.0 (oldest supported accelerate torch version), DTensor is in torch.distributed._tensor + from torch.distributed._tensor import DTensor + + return any(isinstance(p, DTensor) for p in model.parameters()) + + +def extract_model_from_parallel( + model, keep_fp32_wrapper: bool = True, keep_torch_compile: bool = True, recursive: bool = False +): + """ + Extract a model from its distributed containers. + + Args: + model (`torch.nn.Module`): + The model to extract. + keep_fp32_wrapper (`bool`, *optional*): + Whether to remove mixed precision hooks from the model. + keep_torch_compile (`bool`, *optional*): + Whether to unwrap compiled model. + recursive (`bool`, *optional*, defaults to `False`): + Whether to recursively extract all cases of `module.module` from `model` as well as unwrap child sublayers + recursively, not just the top-level distributed containers. + + Returns: + `torch.nn.Module`: The extracted model. + """ + options = (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel) + + is_compiled = is_compiled_module(model) + has_compiled = has_compiled_regions(model) + + compiled_model = None + if is_compiled: + compiled_model = model + model = model._orig_mod + elif has_compiled: + # Skip if top-level not compiled, subs stay wrapped + if "_orig_mod" in model.__dict__: + compiled_model = model + model = model.__dict__["_orig_mod"] + + if is_deepspeed_available(): + from deepspeed import DeepSpeedEngine + + options += (DeepSpeedEngine,) + + if is_torch_version(">=", FSDP_PYTORCH_VERSION) and is_torch_distributed_available(): + from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP + + options += (FSDP,) + + while isinstance(model, options): + model = model.module + + if recursive: + # This is needed in cases such as using FSDPv2 on XLA + def _recursive_unwrap(module): + # Wrapped modules are standardly wrapped as `module`, similar to the cases earlier + # with DDP, DataParallel, DeepSpeed, and FSDP + if hasattr(module, "module"): + unwrapped_module = _recursive_unwrap(module.module) + else: + unwrapped_module = module + # Next unwrap child sublayers recursively + for name, child in unwrapped_module.named_children(): + setattr(unwrapped_module, name, _recursive_unwrap(child)) + return unwrapped_module + + # Start with top-level + model = _recursive_unwrap(model) + + if not keep_fp32_wrapper: + forward = model.forward + original_forward = model.__dict__.pop("_original_forward", None) + if original_forward is not None: + while hasattr(forward, "__wrapped__"): + forward = forward.__wrapped__ + if forward == original_forward: + break + model.forward = MethodType(forward, model) + if getattr(model, "_converted_to_transformer_engine", False): + convert_model(model, to_transformer_engine=False) + + if keep_torch_compile and compiled_model is not None: + if is_compiled: + compiled_model._orig_mod = model + model = compiled_model + elif has_compiled: + compiled_model.__dict__["_orig_mod"] = model + model = compiled_model + + return model + + +def wait_for_everyone(): + """ + Introduces a blocking point in the script, making sure all processes have reached this point before continuing. + + + + Make sure all processes will reach this instruction otherwise one of your processes will hang forever. + + + """ + PartialState().wait_for_everyone() + + +def clean_state_dict_for_safetensors(state_dict: dict): + """ + Cleans the state dictionary from a model and removes tensor aliasing if present. + + Args: + state_dict (`dict`): + The state dictionary from a model + """ + ptrs = collections.defaultdict(list) + # When bnb serialization is used, weights in state dict can be strings + for name, tensor in state_dict.items(): + if not isinstance(tensor, str): + ptrs[id_tensor_storage(tensor)].append(name) + + # These are all pointers of tensors with shared memory + shared_ptrs = {ptr: names for ptr, names in ptrs.items() if len(names) > 1} + warn_names = set() + for names in shared_ptrs.values(): + # When not all duplicates have been cleaned, we still remove those keys but put a clear warning. + # If the link between tensors was done at runtime then `from_pretrained` will not get + # the key back leading to random tensor. A proper warning will be shown + # during reload (if applicable), but since the file is not necessarily compatible with + # the config, better show a proper warning. + found_names = [name for name in names if name in state_dict] + warn_names.update(found_names[1:]) + for name in found_names[1:]: + del state_dict[name] + if len(warn_names) > 0: + logger.warning( + f"Removed shared tensor {warn_names} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading", + ) + state_dict = {k: v.contiguous() if isinstance(v, torch.Tensor) else v for k, v in state_dict.items()} + return state_dict + + +def save(obj, f, save_on_each_node: bool = False, safe_serialization: bool = False): + """ + Save the data to disk. Use in place of `torch.save()`. + + Args: + obj: + The data to save + f: + The file (or file-like object) to use to save the data + save_on_each_node (`bool`, *optional*, defaults to `False`): + Whether to only save on the global main process + safe_serialization (`bool`, *optional*, defaults to `False`): + Whether to save `obj` using `safetensors` or the traditional PyTorch way (that uses `pickle`). + """ + # When TorchXLA is enabled, it's necessary to transfer all data to the CPU before saving. + # Another issue arises with `id_tensor_storage`, which treats all XLA tensors as identical. + # If tensors remain on XLA, calling `clean_state_dict_for_safetensors` will result in only + # one XLA tensor remaining. + if PartialState().distributed_type == DistributedType.XLA: + obj = xm._maybe_convert_to_cpu(obj) + # Check if it's a model and remove duplicates + if safe_serialization: + save_func = partial(safe_save_file, metadata={"format": "pt"}) + if isinstance(obj, OrderedDict): + obj = clean_state_dict_for_safetensors(obj) + else: + save_func = torch.save + + if PartialState().is_main_process and not save_on_each_node: + save_func(obj, f) + elif PartialState().is_local_main_process and save_on_each_node: + save_func(obj, f) + + +# The following are considered "safe" globals to reconstruct various types of objects when using `weights_only=True` +# These should be added and then removed after loading in the file +np_core = np._core if is_numpy_available("2.0.0") else np.core +TORCH_SAFE_GLOBALS = [ + # numpy arrays are just numbers, not objects, so we can reconstruct them safely + np_core.multiarray._reconstruct, + np.ndarray, + # The following are needed for the RNG states + encode, + np.dtype, +] + +if is_numpy_available("1.25.0"): + TORCH_SAFE_GLOBALS.append(np.dtypes.UInt32DType) + + +def load(f, map_location=None, **kwargs): + """ + Compatible drop-in replacement of `torch.load()` which allows for `weights_only` to be used if `torch` version is + 2.4.0 or higher. Otherwise will ignore the kwarg. + + Will also add (and then remove) an exception for numpy arrays + + Args: + f: + The file (or file-like object) to use to load the data + map_location: + a function, `torch.device`, string or a dict specifying how to remap storage locations + **kwargs: + Additional keyword arguments to pass to `torch.load()`. + """ + try: + if is_weights_only_available(): + old_safe_globals = torch.serialization.get_safe_globals() + if "weights_only" not in kwargs: + kwargs["weights_only"] = True + torch.serialization.add_safe_globals(TORCH_SAFE_GLOBALS) + else: + kwargs.pop("weights_only", None) + loaded_obj = torch.load(f, map_location=map_location, **kwargs) + finally: + if is_weights_only_available(): + torch.serialization.clear_safe_globals() + if old_safe_globals: + torch.serialization.add_safe_globals(old_safe_globals) + return loaded_obj + + +def get_pretty_name(obj): + """ + Gets a pretty name from `obj`. + """ + if not hasattr(obj, "__qualname__") and not hasattr(obj, "__name__"): + obj = getattr(obj, "__class__", obj) + if hasattr(obj, "__qualname__"): + return obj.__qualname__ + if hasattr(obj, "__name__"): + return obj.__name__ + return str(obj) + + +def merge_dicts(source, destination): + """ + Recursively merges two dictionaries. + + Args: + source (`dict`): The dictionary to merge into `destination`. + destination (`dict`): The dictionary to merge `source` into. + """ + for key, value in source.items(): + if isinstance(value, dict): + node = destination.setdefault(key, {}) + merge_dicts(value, node) + else: + destination[key] = value + + return destination + + +def is_port_in_use(port: Optional[int] = None) -> bool: + """ + Checks if a port is in use on `localhost`. Useful for checking if multiple `accelerate launch` commands have been + run and need to see if the port is already in use. + """ + if port is None: + port = 29500 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(("localhost", port)) == 0 + + +def get_free_port() -> int: + """ + Gets a free port on `localhost`. Useful for automatic port selection when port 0 is specified in distributed + training scenarios. + + Returns: + int: An available port number + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) # bind to port 0 for OS to assign a free port + return s.getsockname()[1] + + +def convert_bytes(size): + "Converts `size` from bytes to the largest possible unit" + for x in ["bytes", "KB", "MB", "GB", "TB"]: + if size < 1024.0: + return f"{round(size, 2)} {x}" + size /= 1024.0 + + return f"{round(size, 2)} PB" + + +def check_os_kernel(): + """Warns if the kernel version is below the recommended minimum on Linux.""" + # see issue #1929 + info = platform.uname() + system = info.system + if system != "Linux": + return + + _, version, *_ = re.split(r"(\d+\.\d+\.\d+)", info.release) + min_version = "5.5.0" + if Version(version) < Version(min_version): + msg = ( + f"Detected kernel version {version}, which is below the recommended minimum of {min_version}; this can " + "cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher." + ) + logger.warning(msg, main_process_only=True) + + +def recursive_getattr(obj, attr: str): + """ + Recursive `getattr`. + + Args: + obj: + A class instance holding the attribute. + attr (`str`): + The attribute that is to be retrieved, e.g. 'attribute1.attribute2'. + """ + + def _getattr(obj, attr): + return getattr(obj, attr) + + return reduce(_getattr, [obj] + attr.split(".")) + + +def get_module_children_bottom_up(model: torch.nn.Module, return_fqns: bool = False) -> list[torch.nn.Module]: + """Traverse the model in bottom-up order and return the children modules in that order. + + Args: + model (`torch.nn.Module`): the model to get the children of + + Returns: + `list[torch.nn.Module]`: a list of children modules of `model` in bottom-up order. The last element is the + `model` itself. + """ + top = model if not return_fqns else ("", model) + stack = [top] + ordered_modules = [] + while stack: + current_module = stack.pop() + if return_fqns: + current_module_name, current_module = current_module + for name, attr in current_module.named_children(): + if isinstance(attr, torch.nn.Module): + if return_fqns: + child_name = current_module_name + "." + name if current_module_name else name + stack.append((child_name, attr)) + else: + stack.append(attr) + if return_fqns: + ordered_modules.append((current_module_name, current_module)) + else: + ordered_modules.append(current_module) + return ordered_modules[::-1] diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/random.py b/kbidu_env/Lib/site-packages/accelerate/utils/random.py new file mode 100644 index 0000000000000000000000000000000000000000..49639b417dfa41259223c468d6f56b5f7926502c --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/random.py @@ -0,0 +1,165 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +from typing import Optional, Union + +import numpy as np +import torch + +from ..state import AcceleratorState +from .constants import CUDA_DISTRIBUTED_TYPES +from .dataclasses import DistributedType, RNGType +from .imports import ( + is_hpu_available, + is_mlu_available, + is_musa_available, + is_neuron_available, + is_npu_available, + is_sdaa_available, + is_torch_xla_available, + is_xpu_available, +) + + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + +def set_seed(seed: int, device_specific: bool = False, deterministic: bool = False): + """ + Helper function for reproducible behavior to set the seed in `random`, `numpy`, `torch`. + + Args: + seed (`int`): + The seed to set. + device_specific (`bool`, *optional*, defaults to `False`): + Whether to differ the seed on each device slightly with `self.process_index`. + deterministic (`bool`, *optional*, defaults to `False`): + Whether to use deterministic algorithms where available. Can slow down training. + """ + if device_specific: + seed += AcceleratorState().process_index + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if is_xpu_available(): + torch.xpu.manual_seed_all(seed) + elif is_npu_available(): + torch.npu.manual_seed_all(seed) + elif is_mlu_available(): + torch.mlu.manual_seed_all(seed) + elif is_sdaa_available(): + torch.sdaa.manual_seed_all(seed) + elif is_musa_available(): + torch.musa.manual_seed_all(seed) + elif is_hpu_available(): + torch.hpu.manual_seed_all(seed) + elif is_neuron_available(): + torch.neuron.manual_seed_all(seed) + else: + torch.cuda.manual_seed_all(seed) + # ^^ safe to call this function even if cuda is not available + if is_torch_xla_available(): + xm.set_rng_state(seed) + + if deterministic: + torch.use_deterministic_algorithms(True) + + +def synchronize_rng_state(rng_type: Optional[RNGType] = None, generator: Optional[torch.Generator] = None): + # Get the proper rng state + if rng_type == RNGType.TORCH: + rng_state = torch.get_rng_state() + elif rng_type == RNGType.CUDA: + rng_state = torch.cuda.get_rng_state() + elif rng_type == RNGType.XLA: + assert is_torch_xla_available(), "Can't synchronize XLA seeds as torch_xla is unavailable." + rng_state = torch.tensor(xm.get_rng_state()) + elif rng_type == RNGType.NPU: + assert is_npu_available(), "Can't synchronize NPU seeds on an environment without NPUs." + rng_state = torch.npu.get_rng_state() + elif rng_type == RNGType.MLU: + assert is_mlu_available(), "Can't synchronize MLU seeds on an environment without MLUs." + rng_state = torch.mlu.get_rng_state() + elif rng_type == RNGType.SDAA: + assert is_sdaa_available(), "Can't synchronize SDAA seeds on an environment without SDAAs." + rng_state = torch.sdaa.get_rng_state() + elif rng_type == RNGType.MUSA: + assert is_musa_available(), "Can't synchronize MUSA seeds on an environment without MUSAs." + rng_state = torch.musa.get_rng_state() + elif rng_type == RNGType.XPU: + assert is_xpu_available(), "Can't synchronize XPU seeds on an environment without XPUs." + rng_state = torch.xpu.get_rng_state() + elif rng_type == RNGType.HPU: + assert is_hpu_available(), "Can't synchronize HPU seeds on an environment without HPUs." + rng_state = torch.hpu.get_rng_state() + elif rng_type == RNGType.NEURON: + assert is_neuron_available(), "Can't synchronize Neuron seeds on an environment without Neuron Cores." + rng_state = torch.neuron.get_rng_state() + elif rng_type == RNGType.GENERATOR: + assert generator is not None, "Need a generator to synchronize its seed." + rng_state = generator.get_state() + + # Broadcast the rng state from device 0 to other devices + state = AcceleratorState() + if state.distributed_type == DistributedType.XLA: + rng_state = rng_state.to(xm.xla_device()) + xm.collective_broadcast([rng_state]) + xm.mark_step() + rng_state = rng_state.cpu() + elif ( + state.distributed_type in CUDA_DISTRIBUTED_TYPES + or state.distributed_type == DistributedType.MULTI_MLU + or state.distributed_type == DistributedType.MULTI_SDAA + or state.distributed_type == DistributedType.MULTI_MUSA + or state.distributed_type == DistributedType.MULTI_NPU + or state.distributed_type == DistributedType.MULTI_XPU + or state.distributed_type == DistributedType.MULTI_HPU + or state.distributed_type == DistributedType.MULTI_NEURON + ): + rng_state = rng_state.to(state.device) + torch.distributed.broadcast(rng_state, 0) + rng_state = rng_state.cpu() + elif state.distributed_type == DistributedType.MULTI_CPU: + torch.distributed.broadcast(rng_state, 0) + + # Set the broadcast rng state + if rng_type == RNGType.TORCH: + torch.set_rng_state(rng_state) + elif rng_type == RNGType.CUDA: + torch.cuda.set_rng_state(rng_state) + elif rng_type == RNGType.NPU: + torch.npu.set_rng_state(rng_state) + elif rng_type == RNGType.MLU: + torch.mlu.set_rng_state(rng_state) + elif rng_type == RNGType.SDAA: + torch.sdaa.set_rng_state(rng_state) + elif rng_type == RNGType.MUSA: + torch.musa.set_rng_state(rng_state) + elif rng_type == RNGType.XPU: + torch.xpu.set_rng_state(rng_state) + elif rng_type == RNGType.HPU: + torch.hpu.set_rng_state(rng_state) + elif rng_type == RNGType.NEURON: + torch.neuron.set_rng_state(rng_state) + elif rng_type == RNGType.XLA: + xm.set_rng_state(rng_state.item()) + elif rng_type == RNGType.GENERATOR: + generator.set_state(rng_state) + + +def synchronize_rng_states(rng_types: list[Union[str, RNGType]], generator: Optional[torch.Generator] = None): + for rng_type in rng_types: + synchronize_rng_state(RNGType(rng_type), generator=generator) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/rich.py b/kbidu_env/Lib/site-packages/accelerate/utils/rich.py new file mode 100644 index 0000000000000000000000000000000000000000..2d48661b7fcef92ef1168b74cc275c6d3ccc67a1 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/rich.py @@ -0,0 +1,24 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .imports import is_rich_available + + +if is_rich_available(): + from rich.traceback import install + + install(show_locals=False) + +else: + raise ModuleNotFoundError("To use the rich extension, install rich with `pip install rich`") diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/torch_xla.py b/kbidu_env/Lib/site-packages/accelerate/utils/torch_xla.py new file mode 100644 index 0000000000000000000000000000000000000000..140133926c2f88d39c70f5a9f46a08f88bed36da --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/torch_xla.py @@ -0,0 +1,51 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.metadata +import subprocess +import sys + + +def install_xla(upgrade: bool = False): + """ + Helper function to install appropriate xla wheels based on the `torch` version in Google Colaboratory. + + Args: + upgrade (`bool`, *optional*, defaults to `False`): + Whether to upgrade `torch` and install the latest `torch_xla` wheels. + + Example: + + ```python + >>> from accelerate.utils import install_xla + + >>> install_xla(upgrade=True) + ``` + """ + in_colab = False + if "IPython" in sys.modules: + in_colab = "google.colab" in str(sys.modules["IPython"].get_ipython()) + + if in_colab: + if upgrade: + torch_install_cmd = ["pip", "install", "-U", "torch"] + subprocess.run(torch_install_cmd, check=True) + # get the current version of torch + torch_version = importlib.metadata.version("torch") + torch_version_trunc = torch_version[: torch_version.rindex(".")] + xla_wheel = f"https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-{torch_version_trunc}-cp37-cp37m-linux_x86_64.whl" + xla_install_cmd = ["pip", "install", xla_wheel] + subprocess.run(xla_install_cmd, check=True) + else: + raise RuntimeError("`install_xla` utility works only on google colab.") diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/tqdm.py b/kbidu_env/Lib/site-packages/accelerate/utils/tqdm.py new file mode 100644 index 0000000000000000000000000000000000000000..2d4873c1573eb2ee7392162f440a76d4f07cd8ce --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/tqdm.py @@ -0,0 +1,43 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .imports import is_tqdm_available + + +if is_tqdm_available(): + from tqdm.auto import tqdm as _tqdm + +from ..state import PartialState + + +def tqdm(*args, main_process_only: bool = True, **kwargs): + """ + Wrapper around `tqdm.tqdm` that optionally displays only on the main process. + + Args: + main_process_only (`bool`, *optional*): + Whether to display the progress bar only on the main process + """ + if not is_tqdm_available(): + raise ImportError("Accelerate's `tqdm` module requires `tqdm` to be installed. Please run `pip install tqdm`.") + if len(args) > 0 and isinstance(args[0], bool): + raise ValueError( + "Passing `True` or `False` as the first argument to Accelerate's `tqdm` wrapper is unsupported. " + "Please use the `main_process_only` keyword argument instead." + ) + disable = kwargs.pop("disable", False) + if main_process_only and not disable: + disable = PartialState().local_process_index != 0 + return _tqdm(*args, **kwargs, disable=disable) diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/transformer_engine.py b/kbidu_env/Lib/site-packages/accelerate/utils/transformer_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..31b61364fe2630f2b03a0ac64749333b14f6eada --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/transformer_engine.py @@ -0,0 +1,186 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from types import MethodType + +import torch.nn as nn + +from .imports import is_hpu_available, is_transformer_engine_available +from .operations import GatheredParameters + + +# Do not import `transformer_engine` at package level to avoid potential issues + + +def convert_model(model, to_transformer_engine=True, _convert_linear=True, _convert_ln=True): + """ + Recursively converts the linear and layernorm layers of a model to their `transformers_engine` counterpart. + """ + if not is_transformer_engine_available(): + raise ImportError("Using `convert_model` requires transformer_engine to be installed.") + + if is_hpu_available(): + import intel_transformer_engine as te + + if not hasattr(te, "LayerNorm"): + # HPU does not have a LayerNorm implementation in TE + te.LayerNorm = nn.LayerNorm + else: + import transformer_engine.pytorch as te + + for name, module in model.named_children(): + if isinstance(module, nn.Linear) and to_transformer_engine and _convert_linear: + has_bias = module.bias is not None + params_to_gather = [module.weight] + if has_bias: + params_to_gather.append(module.bias) + + with GatheredParameters(params_to_gather, modifier_rank=0): + if any(p % 16 != 0 for p in module.weight.shape): + return + te_module = te.Linear( + module.in_features, module.out_features, bias=has_bias, params_dtype=module.weight.dtype + ) + te_module.weight.copy_(module.weight) + if has_bias: + te_module.bias.copy_(module.bias) + + setattr(model, name, te_module) + # Note: @xrsrke (Phuc) found that te.LayerNorm doesn't have any real memory savings or speedups over nn.LayerNorm + elif isinstance(module, nn.LayerNorm) and to_transformer_engine and _convert_ln: + with GatheredParameters([module.weight, module.bias], modifier_rank=0): + has_bias = module.bias is not None + te_module = te.LayerNorm(module.normalized_shape[0], eps=module.eps, params_dtype=module.weight.dtype) + te_module.weight.copy_(module.weight) + if has_bias: + te_module.bias.copy_(module.bias) + + setattr(model, name, te_module) + elif isinstance(module, te.Linear) and not to_transformer_engine and _convert_linear: + has_bias = module.bias is not None + new_module = nn.Linear( + module.in_features, module.out_features, bias=has_bias, params_dtype=module.weight.dtype + ) + new_module.weight.copy_(module.weight) + if has_bias: + new_module.bias.copy_(module.bias) + + setattr(model, name, new_module) + elif isinstance(module, te.LayerNorm) and not to_transformer_engine and _convert_ln: + new_module = nn.LayerNorm(module.normalized_shape[0], eps=module.eps, params_dtype=module.weight.dtype) + new_module.weight.copy_(module.weight) + new_module.bias.copy_(module.bias) + + setattr(model, name, new_module) + else: + convert_model( + module, + to_transformer_engine=to_transformer_engine, + _convert_linear=_convert_linear, + _convert_ln=_convert_ln, + ) + + +def has_transformer_engine_layers(model): + """ + Returns whether a given model has some `transformer_engine` layer or not. + """ + if not is_transformer_engine_available(): + raise ImportError("Using `has_transformer_engine_layers` requires transformer_engine to be installed.") + + if is_hpu_available(): + import intel_transformer_engine as te + + module_cls_to_check = te.Linear + else: + import transformer_engine.pytorch as te + + module_cls_to_check = (te.LayerNorm, te.Linear, te.TransformerLayer) + + for m in model.modules(): + if isinstance(m, module_cls_to_check): + return True + + return False + + +def contextual_fp8_autocast(model_forward, fp8_recipe, use_during_eval=False): + """ + Wrapper for a model's forward method to apply FP8 autocast. Is context aware, meaning that by default it will + disable FP8 autocast during eval mode, which is generally better for more accurate metrics. + """ + if not is_transformer_engine_available(): + raise ImportError("Using `contextual_fp8_autocast` requires transformer_engine to be installed.") + + if is_hpu_available(): + from intel_transformer_engine import fp8_autocast + else: + from transformer_engine.pytorch import fp8_autocast + + def forward(self, *args, **kwargs): + enabled = use_during_eval or self.training + with fp8_autocast(enabled=enabled, fp8_recipe=fp8_recipe): + return model_forward(*args, **kwargs) + + # To act like a decorator so that it can be popped when doing `extract_model_from_parallel` + forward.__wrapped__ = model_forward + + return forward + + +def apply_fp8_autowrap(model, fp8_recipe_handler): + """ + Applies FP8 context manager to the model's forward method + """ + if not is_transformer_engine_available(): + raise ImportError("Using `apply_fp8_autowrap` requires transformer_engine to be installed.") + + if is_hpu_available(): + import intel_transformer_engine.recipe as te_recipe + + is_fp8_block_scaling_available = False + message = "MXFP8 block scaling is not available on HPU." + + else: + import transformer_engine.common.recipe as te_recipe + from transformer_engine.pytorch.fp8 import check_mxfp8_support + + is_fp8_block_scaling_available, message = check_mxfp8_support() + + kwargs = fp8_recipe_handler.to_kwargs() if fp8_recipe_handler is not None else {} + if "fp8_format" in kwargs: + kwargs["fp8_format"] = getattr(te_recipe.Format, kwargs["fp8_format"]) + use_during_eval = kwargs.pop("use_autocast_during_eval", False) + use_mxfp8_block_scaling = kwargs.pop("use_mxfp8_block_scaling", False) + + if use_mxfp8_block_scaling and not is_fp8_block_scaling_available: + raise ValueError(f"MXFP8 block scaling is not available: {message}") + + if use_mxfp8_block_scaling: + if "amax_compute_algo" in kwargs: + raise ValueError("`amax_compute_algo` is not supported for MXFP8 block scaling.") + if "amax_history_len" in kwargs: + raise ValueError("`amax_history_len` is not supported for MXFP8 block scaling.") + fp8_recipe = te_recipe.MXFP8BlockScaling(**kwargs) + else: + fp8_recipe = te_recipe.DelayedScaling(**kwargs) + + new_forward = contextual_fp8_autocast(model.forward, fp8_recipe, use_during_eval) + + if hasattr(model.forward, "__func__"): + model.forward = MethodType(new_forward, model) + else: + model.forward = new_forward + + return model diff --git a/kbidu_env/Lib/site-packages/accelerate/utils/versions.py b/kbidu_env/Lib/site-packages/accelerate/utils/versions.py new file mode 100644 index 0000000000000000000000000000000000000000..985c918f0e057bacc70c372f6906071bb73db577 --- /dev/null +++ b/kbidu_env/Lib/site-packages/accelerate/utils/versions.py @@ -0,0 +1,56 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.metadata +from typing import Union + +from packaging.version import Version, parse + +from .constants import STR_OPERATION_TO_FUNC + + +torch_version = parse(importlib.metadata.version("torch")) + + +def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str): + """ + Compares a library version to some requirement using a given operation. + + Args: + library_or_version (`str` or `packaging.version.Version`): + A library name or a version to check. + operation (`str`): + A string representation of an operator, such as `">"` or `"<="`. + requirement_version (`str`): + The version to compare the library version against + """ + if operation not in STR_OPERATION_TO_FUNC.keys(): + raise ValueError(f"`operation` must be one of {list(STR_OPERATION_TO_FUNC.keys())}, received {operation}") + operation = STR_OPERATION_TO_FUNC[operation] + if isinstance(library_or_version, str): + library_or_version = parse(importlib.metadata.version(library_or_version)) + return operation(library_or_version, parse(requirement_version)) + + +def is_torch_version(operation: str, version: str): + """ + Compares the current PyTorch version to a given reference with an operation. + + Args: + operation (`str`): + A string representation of an operator, such as `">"` or `"<="` + version (`str`): + A string version of PyTorch + """ + return compare_versions(torch_version, operation, version) diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/INSTALLER b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/METADATA b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..c8a61b093c877752873e1eea4ae05af62b0171e3 --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/METADATA @@ -0,0 +1,123 @@ +Metadata-Version: 2.4 +Name: aiohappyeyeballs +Version: 2.6.2 +Summary: Happy Eyeballs for asyncio +License: PSF-2.0 +License-File: LICENSE +Author: J. Nick Koston +Author-email: nick@koston.org +Requires-Python: >=3.10 +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: Operating System :: OS Independent +Classifier: Topic :: Software Development :: Libraries +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: License :: OSI Approved :: Python Software Foundation License +Project-URL: Bug Tracker, https://github.com/aio-libs/aiohappyeyeballs/issues +Project-URL: Changelog, https://github.com/aio-libs/aiohappyeyeballs/blob/main/CHANGELOG.md +Project-URL: Documentation, https://aiohappyeyeballs.readthedocs.io +Project-URL: Repository, https://github.com/aio-libs/aiohappyeyeballs +Description-Content-Type: text/markdown + +# aiohappyeyeballs + +

+ + CI Status + + + Documentation Status + + + Test coverage percentage + +

+

+ + Poetry + + + Ruff + + + pre-commit + +

+

+ + PyPI Version + + Supported Python versions + License +

+ +--- + +**Documentation**: https://aiohappyeyeballs.readthedocs.io + +**Source Code**: https://github.com/aio-libs/aiohappyeyeballs + +--- + +[Happy Eyeballs](https://en.wikipedia.org/wiki/Happy_Eyeballs) +([RFC 8305](https://www.rfc-editor.org/rfc/rfc8305.html)) + +## Use case + +This library exists to allow connecting with +[Happy Eyeballs](https://en.wikipedia.org/wiki/Happy_Eyeballs) +([RFC 8305](https://www.rfc-editor.org/rfc/rfc8305.html)) +when you +already have a list of addrinfo and not a DNS name. + +The stdlib version of `loop.create_connection()` +will only work when you pass in an unresolved name which +is not a good fit when using DNS caching or resolving +names via another method such as `zeroconf`. + +## Installation + +Install this via pip (or your favourite package manager): + +`pip install aiohappyeyeballs` + +## License + +[aiohappyeyeballs is licensed under the same terms as cpython itself.](https://github.com/python/cpython/blob/main/LICENSE) + +## Example usage + +```python + +addr_infos = await loop.getaddrinfo("example.org", 80) + +socket = await start_connection(addr_infos) +socket = await start_connection(addr_infos, local_addr_infos=local_addr_infos, happy_eyeballs_delay=0.2) + +transport, protocol = await loop.create_connection( + MyProtocol, sock=socket, ...) + +# Remove the first address for each family from addr_info +pop_addr_infos_interleave(addr_info, 1) + +# Remove all matching address from addr_info +remove_addr_infos(addr_info, "dead::beef::") + +# Convert a local_addr to local_addr_infos +local_addr_infos = addr_to_addr_infos(("127.0.0.1",0)) +``` + +## Credits + +This package contains code from cpython and is licensed under the same terms as cpython itself. + +This package was created with +[Copier](https://copier.readthedocs.io/) and the +[browniebroke/pypackage-template](https://github.com/browniebroke/pypackage-template) +project template. + diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/RECORD b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..585d8db805fd56c39bffcd60ce5c67a04b3d160d --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/RECORD @@ -0,0 +1,16 @@ +aiohappyeyeballs-2.6.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +aiohappyeyeballs-2.6.2.dist-info/METADATA,sha256=cqs2VY8TwE2e_4qqnW409Si3suvj-aM-8dCiWG2Angk,5888 +aiohappyeyeballs-2.6.2.dist-info/RECORD,, +aiohappyeyeballs-2.6.2.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88 +aiohappyeyeballs-2.6.2.dist-info/licenses/LICENSE,sha256=Oy-B_iHRgcSZxZolbI4ZaEVdZonSaaqFNzv7avQdo78,13936 +aiohappyeyeballs/__init__.py,sha256=Af9ADZj3BWLfGaA7ITOFPlKIenh3ozSNWL3yezSZ2Jw,361 +aiohappyeyeballs/__pycache__/__init__.cpython-314.pyc,, +aiohappyeyeballs/__pycache__/_staggered.cpython-314.pyc,, +aiohappyeyeballs/__pycache__/impl.cpython-314.pyc,, +aiohappyeyeballs/__pycache__/types.cpython-314.pyc,, +aiohappyeyeballs/__pycache__/utils.cpython-314.pyc,, +aiohappyeyeballs/_staggered.py,sha256=aj3cSwHEDX88UMfO9bUau9tfrRAszhjg99dpEMiAOGM,6698 +aiohappyeyeballs/impl.py,sha256=TIkAK4xfACvKBp1s7DAKSobHefUTOC2HGE-n0tOthRk,9667 +aiohappyeyeballs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +aiohappyeyeballs/types.py,sha256=_8JmHFix6MeM1e7hRP7BleEaGy93GswGtzQv068zKY8,288 +aiohappyeyeballs/utils.py,sha256=dPAcNcrU_VhaTolTEEA94hgh9ONDN9_dYT4Xo9ORqQw,2922 diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/WHEEL b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..6a352135a5549b2935f6d31e32ddf0f3f62b7d85 --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: poetry-core 2.4.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/licenses/LICENSE b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f26bcf4d2de6eb136e31006ca3ab447d5e488adf --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs-2.6.2.dist-info/licenses/LICENSE @@ -0,0 +1,279 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see https://opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +Python software and documentation are licensed under the +Python Software Foundation License Version 2. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Version 2 +and the Zero-Clause BSD license. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. + + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; +All Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION +---------------------------------------------------------------------- + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/__init__.py b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ad6bfb5a073f17d54395af7696b8c3fa312e1abb --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__init__.py @@ -0,0 +1,14 @@ +__version__ = "2.6.2" + +from .impl import start_connection +from .types import AddrInfoType, SocketFactoryType +from .utils import addr_to_addr_infos, pop_addr_infos_interleave, remove_addr_infos + +__all__ = ( + "AddrInfoType", + "SocketFactoryType", + "addr_to_addr_infos", + "pop_addr_infos_interleave", + "remove_addr_infos", + "start_connection", +) diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/__init__.cpython-314.pyc b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1fb81594386610a122faebf3287f2b9a51a7bd58 Binary files /dev/null and b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/__init__.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/_staggered.cpython-314.pyc b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/_staggered.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce61482fa887ff070bb034e26dd04862907dfb3c Binary files /dev/null and b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/_staggered.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/impl.cpython-314.pyc b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/impl.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..40774e16a22295288f1ca4d2d0ab8b0b07036727 Binary files /dev/null and b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/impl.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/types.cpython-314.pyc b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/types.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c91aa41574417d78902e0c9c6b29f075694be071 Binary files /dev/null and b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/types.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/utils.cpython-314.pyc b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..567ad40c38109615dbd0a6aefbe78cd983a8354a Binary files /dev/null and b/kbidu_env/Lib/site-packages/aiohappyeyeballs/__pycache__/utils.cpython-314.pyc differ diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/_staggered.py b/kbidu_env/Lib/site-packages/aiohappyeyeballs/_staggered.py new file mode 100644 index 0000000000000000000000000000000000000000..bb745dc0a81460327d6cc19e5bd38cc44f296fe5 --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs/_staggered.py @@ -0,0 +1,197 @@ +import asyncio +import contextlib +from collections.abc import Awaitable, Callable, Iterable +from typing import ( + TYPE_CHECKING, + Any, + TypeVar, +) + +_T = TypeVar("_T") + +RE_RAISE_EXCEPTIONS = (SystemExit, KeyboardInterrupt) + + +def _set_result(wait_next: "asyncio.Future[None]") -> None: + """Set the result of a future if it is not already done.""" + if not wait_next.done(): + wait_next.set_result(None) + + +async def _wait_one( + futures: "Iterable[asyncio.Future[Any]]", + loop: asyncio.AbstractEventLoop, +) -> _T: + """Wait for the first future to complete.""" + wait_next = loop.create_future() + + def _on_completion(fut: "asyncio.Future[Any]") -> None: + if not wait_next.done(): + wait_next.set_result(fut) + + for f in futures: + f.add_done_callback(_on_completion) + + try: + return await wait_next + finally: + for f in futures: + f.remove_done_callback(_on_completion) + + +async def staggered_race( + coro_fns: Iterable[Callable[[], Awaitable[_T]]], + delay: float | None, + *, + loop: asyncio.AbstractEventLoop | None = None, +) -> tuple[_T | None, int | None, list[BaseException | None]]: + """ + Run coroutines with staggered start times and take the first to finish. + + This method takes an iterable of coroutine functions. The first one is + started immediately. From then on, whenever the immediately preceding one + fails (raises an exception), or when *delay* seconds has passed, the next + coroutine is started. This continues until one of the coroutines complete + successfully, in which case all others are cancelled, or until all + coroutines fail. + + The coroutines provided should be well-behaved in the following way: + + * They should only ``return`` if completed successfully. + + * They should always raise an exception if they did not complete + successfully. In particular, if they handle cancellation, they should + probably reraise, like this:: + + try: + # do work + except asyncio.CancelledError: + # undo partially completed work + raise + + Args: + ---- + coro_fns: an iterable of coroutine functions, i.e. callables that + return a coroutine object when called. Use ``functools.partial`` or + lambdas to pass arguments. + + delay: amount of time, in seconds, between starting coroutines. If + ``None``, the coroutines will run sequentially. + + loop: the event loop to use. If ``None``, the running loop is used. + + Returns: + ------- + tuple *(winner_result, winner_index, exceptions)* where + + - *winner_result*: the result of the winning coroutine, or ``None`` + if no coroutines won. + + - *winner_index*: the index of the winning coroutine in + ``coro_fns``, or ``None`` if no coroutines won. If the winning + coroutine may return None on success, *winner_index* can be used + to definitively determine whether any coroutine won. + + - *exceptions*: list of exceptions returned by the coroutines. + ``len(exceptions)`` is equal to the number of coroutines actually + started, and the order is the same as in ``coro_fns``. The winning + coroutine's entry is ``None``. + + """ + loop = loop or asyncio.get_running_loop() + exceptions: list[BaseException | None] = [] + tasks: set[asyncio.Task[tuple[_T, int] | None]] = set() + + async def run_one_coro( + coro_fn: Callable[[], Awaitable[_T]], + this_index: int, + start_next: "asyncio.Future[None]", + ) -> tuple[_T, int] | None: + """ + Run a single coroutine. + + If the coroutine fails, set the exception in the exceptions list and + start the next coroutine by setting the result of the start_next. + + If the coroutine succeeds, return the result and the index of the + coroutine in the coro_fns list. + + If SystemExit or KeyboardInterrupt is raised, re-raise it. + """ + try: + result = await coro_fn() + except RE_RAISE_EXCEPTIONS: + raise + except BaseException as e: + exceptions[this_index] = e + _set_result(start_next) # Kickstart the next coroutine + return None + + return result, this_index + + start_next_timer: asyncio.TimerHandle | None = None + start_next: asyncio.Future[None] | None + task: asyncio.Task[tuple[_T, int] | None] + done: asyncio.Future[None] | asyncio.Task[tuple[_T, int] | None] + coro_iter = iter(coro_fns) + this_index = -1 + try: + while True: + if coro_fn := next(coro_iter, None): + this_index += 1 + exceptions.append(None) + start_next = loop.create_future() + task = loop.create_task(run_one_coro(coro_fn, this_index, start_next)) + tasks.add(task) + start_next_timer = ( + loop.call_later(delay, _set_result, start_next) if delay else None + ) + elif not tasks: + # We exhausted the coro_fns list and no tasks are running + # so we have no winner and all coroutines failed. + break + + while tasks or start_next: + done = await _wait_one( + (*tasks, start_next) if start_next else tasks, loop + ) + if done is start_next: + # The current task has failed or the timer has expired + # so we need to start the next task. + start_next = None + if start_next_timer: + start_next_timer.cancel() + start_next_timer = None + + # Break out of the task waiting loop to start the next + # task. + break + + if TYPE_CHECKING: + assert isinstance(done, asyncio.Task) + + tasks.remove(done) + if winner := done.result(): + return *winner, exceptions + finally: + # We either have: + # - a winner + # - all tasks failed + # - a KeyboardInterrupt or SystemExit. + + # + # If the timer is still running, cancel it. + # + if start_next_timer: + start_next_timer.cancel() + + # + # If there are any tasks left, cancel them and than + # wait them so they fill the exceptions list. + # + for task in tasks: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + return None, None, exceptions diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/impl.py b/kbidu_env/Lib/site-packages/aiohappyeyeballs/impl.py new file mode 100644 index 0000000000000000000000000000000000000000..4112ab4d1b2bdeec6da543eac4433af175699c7d --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs/impl.py @@ -0,0 +1,261 @@ +"""Base implementation.""" + +import asyncio +import collections +import contextlib +import functools +import itertools +import socket +from collections.abc import Sequence + +from . import _staggered +from .types import AddrInfoType, SocketFactoryType + + +async def start_connection( + addr_infos: Sequence[AddrInfoType], + *, + local_addr_infos: Sequence[AddrInfoType] | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + loop: asyncio.AbstractEventLoop | None = None, + socket_factory: SocketFactoryType | None = None, +) -> socket.socket: + """ + Connect to a TCP server. + + Create a socket connection to a specified destination. The + destination is specified as a list of AddrInfoType tuples as + returned from getaddrinfo(). + + The arguments are, in order: + + * ``family``: the address family, e.g. ``socket.AF_INET`` or + ``socket.AF_INET6``. + * ``type``: the socket type, e.g. ``socket.SOCK_STREAM`` or + ``socket.SOCK_DGRAM``. + * ``proto``: the protocol, e.g. ``socket.IPPROTO_TCP`` or + ``socket.IPPROTO_UDP``. + * ``canonname``: the canonical name of the address, e.g. + ``"www.python.org"``. + * ``sockaddr``: the socket address + + This method is a coroutine which will try to establish the connection + in the background. When successful, the coroutine returns a + socket. + + The expected use case is to use this method in conjunction with + loop.create_connection() to establish a connection to a server:: + + socket = await start_connection(addr_infos) + transport, protocol = await loop.create_connection( + MyProtocol, sock=socket, ...) + """ + if not addr_infos: + raise ValueError("addr_infos must not be empty") + + current_loop = loop or asyncio.get_running_loop() + + single_addr_info = len(addr_infos) == 1 + + if happy_eyeballs_delay is not None and interleave is None: + # If using happy eyeballs, default to interleave addresses by family + interleave = 1 + + if interleave and not single_addr_info: + addr_infos = _interleave_addrinfos(addr_infos, interleave) + + sock: socket.socket | None = None + # uvloop can raise RuntimeError instead of OSError + exceptions: list[list[OSError | RuntimeError]] = [] + if happy_eyeballs_delay is None or single_addr_info: + # not using happy eyeballs + for addrinfo in addr_infos: + try: + sock = await _connect_sock( + current_loop, + exceptions, + addrinfo, + local_addr_infos, + None, + socket_factory, + ) + break + except (RuntimeError, OSError): + continue + else: # using happy eyeballs + open_sockets: set[socket.socket] = set() + try: + sock, _, _ = await _staggered.staggered_race( + ( + functools.partial( + _connect_sock, + current_loop, + exceptions, + addrinfo, + local_addr_infos, + open_sockets, + socket_factory, + ) + for addrinfo in addr_infos + ), + happy_eyeballs_delay, + ) + finally: + # If we have a winner, staggered_race will + # cancel the other tasks, however there is a + # small race window where any of the other tasks + # can be done before they are cancelled which + # will leave the socket open. To avoid this problem + # we pass a set to _connect_sock to keep track of + # the open sockets and close them here if there + # are any "runner up" sockets. + for s in open_sockets: + if s is not sock: + with contextlib.suppress(OSError): + s.close() + open_sockets = None # type: ignore[assignment] + + if sock is None: + all_exceptions = [exc for sub in exceptions for exc in sub] + try: + first_exception = all_exceptions[0] + if len(all_exceptions) == 1: + raise first_exception + else: + # If they all have the same str(), raise one. + model = str(first_exception) + if all(str(exc) == model for exc in all_exceptions): + raise first_exception + # Raise a combined exception so the user can see all + # the various error messages. + msg = "Multiple exceptions: {}".format( + ", ".join(str(exc) for exc in all_exceptions) + ) + # If the errno is the same for all exceptions, raise + # an OSError with that errno. + if isinstance(first_exception, OSError): + first_errno = first_exception.errno + if all( + isinstance(exc, OSError) and exc.errno == first_errno + for exc in all_exceptions + ): + raise OSError(first_errno, msg) + elif isinstance(first_exception, RuntimeError) and all( + isinstance(exc, RuntimeError) for exc in all_exceptions + ): + raise RuntimeError(msg) + # We have a mix of OSError and RuntimeError + # so we have to pick which one to raise. + # and we raise OSError for compatibility + raise OSError(msg) + finally: + all_exceptions = None # type: ignore[assignment] + exceptions = None # type: ignore[assignment] + + return sock + + +async def _connect_sock( + loop: asyncio.AbstractEventLoop, + exceptions: list[list[OSError | RuntimeError]], + addr_info: AddrInfoType, + local_addr_infos: Sequence[AddrInfoType] | None = None, + open_sockets: set[socket.socket] | None = None, + socket_factory: SocketFactoryType | None = None, +) -> socket.socket: + """ + Create, bind and connect one socket. + + If open_sockets is passed, add the socket to the set of open sockets. + Any failure caught here will remove the socket from the set and close it. + + Callers can use this set to close any sockets that are not the winner + of all staggered tasks in the result there are runner up sockets aka + multiple winners. + """ + my_exceptions: list[OSError | RuntimeError] = [] + exceptions.append(my_exceptions) + family, type_, proto, _, address = addr_info + sock = None + try: + if socket_factory is not None: + sock = socket_factory(addr_info) + else: + sock = socket.socket(family=family, type=type_, proto=proto) + if open_sockets is not None: + open_sockets.add(sock) + sock.setblocking(False) + if local_addr_infos is not None: + for lfamily, _, _, _, laddr in local_addr_infos: + # skip local addresses of different family + if lfamily != family: + continue + try: + sock.bind(laddr) + break + except OSError as exc: + msg = ( + f"error while attempting to bind on " + f"address {laddr!r}: " + f"{(exc.strerror or '').lower()}" + ) + exc = OSError(exc.errno, msg) + my_exceptions.append(exc) + else: # all bind attempts failed + if my_exceptions: + raise my_exceptions.pop() + else: + raise OSError(f"no matching local address with {family=} found") + await loop.sock_connect(sock, address) + return sock + except (RuntimeError, OSError) as exc: + my_exceptions.append(exc) + if sock is not None: + if open_sockets is not None: + open_sockets.remove(sock) + try: + sock.close() + except OSError as e: + my_exceptions.append(e) + raise + raise + except: + if sock is not None: + if open_sockets is not None: + open_sockets.remove(sock) + try: + sock.close() + except OSError as e: + my_exceptions.append(e) + raise + raise + finally: + exceptions = my_exceptions = None # type: ignore[assignment] + + +def _interleave_addrinfos( + addrinfos: Sequence[AddrInfoType], first_address_family_count: int = 1 +) -> list[AddrInfoType]: + """Interleave list of addrinfo tuples by family.""" + # Group addresses by family + addrinfos_by_family: collections.OrderedDict[int, list[AddrInfoType]] = ( + collections.OrderedDict() + ) + for addr in addrinfos: + family = addr[0] + if family not in addrinfos_by_family: + addrinfos_by_family[family] = [] + addrinfos_by_family[family].append(addr) + addrinfos_lists = list(addrinfos_by_family.values()) + + reordered: list[AddrInfoType] = [] + if first_address_family_count > 1: + reordered.extend(addrinfos_lists[0][: first_address_family_count - 1]) + del addrinfos_lists[0][: first_address_family_count - 1] + reordered.extend( + a + for a in itertools.chain.from_iterable(itertools.zip_longest(*addrinfos_lists)) + if a is not None + ) + return reordered diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/py.typed b/kbidu_env/Lib/site-packages/aiohappyeyeballs/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/types.py b/kbidu_env/Lib/site-packages/aiohappyeyeballs/types.py new file mode 100644 index 0000000000000000000000000000000000000000..80f09afdd6d5f4cca4d9552653867e2de8e2d6a5 --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs/types.py @@ -0,0 +1,14 @@ +"""Types for aiohappyeyeballs.""" + +import socket +from collections.abc import Callable + +AddrInfoType = tuple[ + int | socket.AddressFamily, + int | socket.SocketKind, + int, + str, + tuple, # type: ignore[type-arg] +] + +SocketFactoryType = Callable[[AddrInfoType], socket.socket] diff --git a/kbidu_env/Lib/site-packages/aiohappyeyeballs/utils.py b/kbidu_env/Lib/site-packages/aiohappyeyeballs/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5969691ce3627dd64b9ca539331b3ef4af5da6de --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohappyeyeballs/utils.py @@ -0,0 +1,92 @@ +"""Utility functions for aiohappyeyeballs.""" + +import ipaddress +import socket + +from .types import AddrInfoType + + +def addr_to_addr_infos( + addr: tuple[str, int, int, int] | tuple[str, int, int] | tuple[str, int] | None, +) -> list[AddrInfoType] | None: + """Convert an address tuple to a list of addr_info tuples.""" + if addr is None: + return None + host = addr[0] + port = addr[1] + is_ipv6 = ":" in host + if is_ipv6: + flowinfo = 0 + scopeid = 0 + addr_len = len(addr) + if addr_len >= 4: + scopeid = addr[3] # type: ignore[misc] + if addr_len >= 3: + flowinfo = addr[2] # type: ignore[misc] + addr = (host, port, flowinfo, scopeid) + family = socket.AF_INET6 + else: + addr = (host, port) + family = socket.AF_INET + return [(family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", addr)] + + +def pop_addr_infos_interleave( + addr_infos: list[AddrInfoType], interleave: int | None = None +) -> None: + """ + Pop addr_info from the list of addr_infos by family up to interleave times. + + The interleave parameter is used to know how many addr_infos for + each family should be popped of the top of the list. + """ + seen: dict[int, int] = {} + if interleave is None: + interleave = 1 + to_remove: list[AddrInfoType] = [] + for addr_info in addr_infos: + family = addr_info[0] + if family not in seen: + seen[family] = 0 + if seen[family] < interleave: + to_remove.append(addr_info) + seen[family] += 1 + for addr_info in to_remove: + addr_infos.remove(addr_info) + + +def _addr_tuple_to_ip_address( + addr: tuple[str, int] | tuple[str, int, int, int], +) -> tuple[ipaddress.IPv4Address, int] | tuple[ipaddress.IPv6Address, int, int, int]: + """Convert an address tuple to an IPv4Address.""" + return (ipaddress.ip_address(addr[0]), *addr[1:]) + + +def remove_addr_infos( + addr_infos: list[AddrInfoType], + addr: tuple[str, int] | tuple[str, int, int, int], +) -> None: + """ + Remove an address from the list of addr_infos. + + The addr value is typically the return value of + sock.getpeername(). + """ + bad_addrs_infos: list[AddrInfoType] = [] + for addr_info in addr_infos: + if addr_info[-1] == addr: + bad_addrs_infos.append(addr_info) + if bad_addrs_infos: + for bad_addr_info in bad_addrs_infos: + addr_infos.remove(bad_addr_info) + return + # Slow path in case addr is formatted differently + match_addr = _addr_tuple_to_ip_address(addr) + for addr_info in addr_infos: + if match_addr == _addr_tuple_to_ip_address(addr_info[-1]): + bad_addrs_infos.append(addr_info) + if bad_addrs_infos: + for bad_addr_info in bad_addrs_infos: + addr_infos.remove(bad_addr_info) + return + raise ValueError(f"Address {addr} not found in addr_infos") diff --git a/kbidu_env/Lib/site-packages/aiohttp/.hash/_cparser.pxd.hash b/kbidu_env/Lib/site-packages/aiohttp/.hash/_cparser.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..061bccc5c228b0a588ce21df0c27acfc9ffe2556 --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohttp/.hash/_cparser.pxd.hash @@ -0,0 +1 @@ +535e028ee43275a6ceca0817f6ceb90ab44e8a73f20a3a8f5a0c036447687f79 *D:/a/aiohttp/aiohttp/aiohttp/_cparser.pxd diff --git a/kbidu_env/Lib/site-packages/aiohttp/.hash/_find_header.pxd.hash b/kbidu_env/Lib/site-packages/aiohttp/.hash/_find_header.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..8af9f81d71e4e4ee436f4bf3e6f95940e5f3666e --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohttp/.hash/_find_header.pxd.hash @@ -0,0 +1 @@ +0455129b185e981b5b96ac738f31f7c74dc57f1696953cae0083b3f18679fe73 *D:/a/aiohttp/aiohttp/aiohttp/_find_header.pxd diff --git a/kbidu_env/Lib/site-packages/aiohttp/.hash/_http_parser.pyx.hash b/kbidu_env/Lib/site-packages/aiohttp/.hash/_http_parser.pyx.hash new file mode 100644 index 0000000000000000000000000000000000000000..d621a8b5a0fa43e0f534c8d0ca8c7e8f596466e7 --- /dev/null +++ b/kbidu_env/Lib/site-packages/aiohttp/.hash/_http_parser.pyx.hash @@ -0,0 +1 @@ +00fc87643dd46deca4e9ebe3f7d6cd42e118b5f0581552da5e4c418dc0900cb7 *D:/a/aiohttp/aiohttp/aiohttp/_http_parser.pyx