File size: 11,325 Bytes
c4f8e4a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
import os
import json
import multiprocessing as mp
from multiprocessing import Process, Queue
import time

import torch
import yaml
import h5py
from tqdm import tqdm

import sys
import os

# Add project root to Python path
PROJECT_ROOT = os.environ.get('HRDT_PROJECT_ROOT', os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
sys.path.append(PROJECT_ROOT)
from models.encoder.t5_encoder import T5Embedder

# Get paths from environment variables or use defaults
MODEL_PATH = os.environ.get('T5_MODEL_PATH', "/hfm/t5-v1_1-xxl")
CONFIG_PATH = os.environ.get('HRDT_CONFIG_PATH', os.path.join(PROJECT_ROOT, "configs/hrdt_pretrain.yaml"))
TARGET_DIR = os.environ.get('EGODEX_DATA_ROOT', "/fpv/egodex")

# Multi-process configuration (can be overridden by environment variables)
NUM_GPUS = int(os.environ.get('NUM_GPUS', 1))  # Total number of GPUs
PROCESSES_PER_GPU = int(os.environ.get('PROCESSES_PER_GPU', 4))  # Processes per GPU
TOTAL_PROCESSES = NUM_GPUS * PROCESSES_PER_GPU  # Total processes


def collect_all_files():
    """Collect all files that need to be processed"""
    all_files = []
    
    # Get all directories that need to be processed
    dataset_dirs = []
    for item in os.listdir(TARGET_DIR):
        item_path = os.path.join(TARGET_DIR, item)
        if os.path.isdir(item_path):
            if item in ['extra', 'test'] or item.startswith('part'):
                dataset_dirs.append(item_path)
    
    dataset_dirs.sort()
    
    # Collect all files
    for dataset_dir in dataset_dirs:
        dataset_name = os.path.basename(dataset_dir)
        
        for task_name in os.listdir(dataset_dir):
            task_dir = os.path.join(dataset_dir, task_name)
            if os.path.isdir(task_dir):
                hdf5_files = [f for f in os.listdir(task_dir) if f.endswith('.hdf5')]
                hdf5_files.sort(key=lambda x: int(x.split('.')[0]))
                
                for hdf5_file in hdf5_files:
                    hdf5_path = os.path.join(task_dir, hdf5_file)
                    file_index = hdf5_file.split('.')[0]
                    pt_file = f"{file_index}.pt"
                    pt_path = os.path.join(task_dir, pt_file)
                    
                    # Only add files that haven't been processed yet
                    if not os.path.exists(pt_path):
                        all_files.append({
                            'hdf5_path': hdf5_path,
                            'pt_path': pt_path,
                            'task_name': task_name,
                            'dataset_name': dataset_name,
                            'file_index': file_index
                        })
    
    return all_files


def worker_process(process_id, gpu_id, file_list, progress_queue):
    """Worker process function"""
    try:
        # Set CUDA device
        device = torch.device(f"cuda:{gpu_id}")
        torch.cuda.set_device(device)
        
        # Load configuration
        with open(CONFIG_PATH, "r") as fp:
            config = yaml.safe_load(fp)
        
        # Initialize T5 encoder
        text_embedder = T5Embedder(
            from_pretrained=MODEL_PATH, 
            model_max_length=config["dataset"]["tokenizer_max_length"], 
            device=device
        )
        tokenizer, text_encoder = text_embedder.tokenizer, text_embedder.model
        
        print(f"Process {process_id} (GPU {gpu_id}) starting to process {len(file_list)} files")
        
        processed_count = 0
        failed_count = 0
        
        for file_info in file_list:
            try:
                hdf5_path = file_info['hdf5_path']
                pt_path = file_info['pt_path']
                task_name = file_info['task_name']
                dataset_name = file_info['dataset_name']
                file_index = file_info['file_index']

                # print("L106:", pt_path);exit(0)
                
                # Check again if file already exists (prevent duplicate processing by other processes)
                if os.path.exists(pt_path):
                    processed_count += 1
                    progress_queue.put(('processed', process_id))
                    continue
                
                # Extract language instructions from HDF5 file
                instructions = []
                with h5py.File(hdf5_path, 'r') as f:
                    # Extract main description
                    if 'llm_description' in f.attrs:
                        instruction = f.attrs['llm_description']
                        if isinstance(instruction, bytes):
                            instruction = instruction.decode('utf-8')
                        instructions.append(instruction)
                    
                    # Extract second description (if exists)
                    if 'llm_description2' in f.attrs:
                        instruction2 = f.attrs['llm_description2']
                        if isinstance(instruction2, bytes):
                            instruction2 = instruction2.decode('utf-8')
                        instructions.append(instruction2)
                
                if not instructions:
                    print(f"Process {process_id}: Warning: No language instructions found in {hdf5_path}")
                    failed_count += 1
                    progress_queue.put(('failed', process_id))
                    continue
                
                # Encode language instructions
                tokenized_res = tokenizer(
                    instructions, return_tensors="pt",
                    padding="longest",
                    truncation=True
                )
                tokens = tokenized_res["input_ids"].to(device)
                attn_mask = tokenized_res["attention_mask"].to(device)
                
                with torch.no_grad():
                    text_embeds = text_encoder(
                        input_ids=tokens,
                        attention_mask=attn_mask
                    )["last_hidden_state"].detach().cpu()
                
                attn_mask = attn_mask.cpu().bool()
                
                # Process encoding for each instruction and save
                if len(instructions) == 1:
                    # Only one instruction
                    text_embed = text_embeds[0][attn_mask[0]]
                    torch.save({
                        "instruction": instructions[0],
                        "embeddings": text_embed,
                        "task_name": task_name,
                        "dataset": dataset_name,
                        "file_index": file_index
                    }, pt_path)
                else:
                    # Multiple instructions
                    text_embed_1 = text_embeds[0][attn_mask[0]]
                    text_embed_2 = text_embeds[1][attn_mask[1]] if len(instructions) > 1 else None
                    
                    torch.save({
                        "instruction": instructions[0],
                        "instruction2": instructions[1] if len(instructions) > 1 else None,
                        "embeddings": text_embed_1,
                        "embeddings2": text_embed_2,
                        "task_name": task_name,
                        "dataset": dataset_name,
                        "file_index": file_index
                    }, pt_path)
                
                processed_count += 1
                progress_queue.put(('processed', process_id))
                
            except Exception as e:
                print(f"Process {process_id}: Error: Processing file {file_info['hdf5_path']} encountered problem: {str(e)}")
                failed_count += 1
                progress_queue.put(('failed', process_id))
                continue
        
        print(f"Process {process_id} (GPU {gpu_id}) completed: successful {processed_count}, failed {failed_count}")
        progress_queue.put(('done', process_id, processed_count, failed_count))
        
    except Exception as e:
        print(f"Process {process_id} encountered serious error: {str(e)}")
        progress_queue.put(('error', process_id, str(e)))


def progress_monitor(total_files, progress_queue, num_processes):
    """Progress monitoring function"""
    processed = 0
    failed = 0
    finished_processes = 0
    
    pbar = tqdm(total=total_files, desc="Processing progress")
    
    while finished_processes < num_processes:
        try:
            msg = progress_queue.get(timeout=1)
            
            if msg[0] == 'processed':
                processed += 1
                pbar.update(1)
            elif msg[0] == 'failed':
                failed += 1
                pbar.update(1)
            elif msg[0] == 'done':
                finished_processes += 1
                process_id, proc_processed, proc_failed = msg[1], msg[2], msg[3]
                print(f"\nProcess {process_id} completed: successful {proc_processed}, failed {proc_failed}")
            elif msg[0] == 'error':
                finished_processes += 1
                process_id, error_msg = msg[1], msg[2]
                print(f"\nProcess {process_id} error: {error_msg}")
                
        except:
            # Timeout, continue waiting
            continue
    
    pbar.close()
    return processed, failed


def main():
    print("Starting to collect all files to be processed...")
    all_files = collect_all_files()
    
    if not all_files:
        print("No files found to process!")
        return
    
    print(f"Found files to process")
    
    # Distribute files to different processes
    files_per_process = len(all_files) // TOTAL_PROCESSES
    file_lists = []
    
    for i in range(TOTAL_PROCESSES):
        start_idx = i * files_per_process
        if i == TOTAL_PROCESSES - 1:  # Last process handles all remaining files
            end_idx = len(all_files)
        else:
            end_idx = start_idx + files_per_process
        
        file_lists.append(all_files[start_idx:end_idx])
    
    print(f"Will use {NUM_GPUS} GPUs with {PROCESSES_PER_GPU} processes each, total {TOTAL_PROCESSES} processes")
    for i, file_list in enumerate(file_lists):
        gpu_id = i // PROCESSES_PER_GPU
        print(f"Process {i} (GPU {gpu_id}): {len(file_list)} files")
    
    # Create progress queue
    progress_queue = Queue()
    
    # Start progress monitoring process
    monitor_process = Process(
        target=progress_monitor, 
        args=(len(all_files), progress_queue, TOTAL_PROCESSES)
    )
    monitor_process.start()
    
    # Create and start worker processes
    processes = []
    for i in range(TOTAL_PROCESSES):
        gpu_id = i // PROCESSES_PER_GPU  # Calculate GPU ID
        process = Process(
            target=worker_process,
            args=(i, gpu_id, file_lists[i], progress_queue)
        )
        process.start()
        processes.append(process)
        time.sleep(0.5)  # Slight delay to avoid GPU initialization conflicts
    
    # Wait for all processes to complete
    for process in processes:
        process.join()
    
    # Wait for progress monitoring to complete
    monitor_process.join()
    
    print("\nAll processes completed!")


if __name__ == "__main__":
    # Set multi-process start method
    mp.set_start_method('spawn', force=True)
    main()