[auto] Upload folder Recommend/dataset

Recommend/dataset/downloader_readme.md

@@ -0,0 +1,17 @@
+# PPD 这个项目只关注Lamp数据集，风格化实验只在Lamp 4, 5, 7上做
+
+
+‘’‘
+已下载的文件
+LaMP 5: Personalized Scholarly Title Generation
+data/lamp/LaMP_5/user/train/: 训练集（9682 条 questions + 9682 条 outputs）
+data/lamp/LaMP_5/user/dev/: 验证集（2500 条 questions + 2500 条 outputs）
+data/lamp/LaMP_5/user/test/: 测试集（2500 条 questions）
+LaMP 7: Personalized Tweet Paraphrasing
+data/lamp/LaMP_7/user/train/: 训练集（10437 条 questions + 10437 条 outputs）
+data/lamp/LaMP_7/user/dev/: 验证集（1500 条 questions + 1500 条 outputs）
+data/lamp/LaMP_7/user/test/: 测试集（1496 条 questions）
+数据格式
+questions 文件：包含 id, input, profile 字段
+outputs 文件：包含 task 和 golds 字段，其中 golds 是包含 id 和 output 的列表
+’‘’

Recommend/dataset/lamp_downloader.py

@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+LaMP 数据集下载脚本
+用于下载 LaMP 任务 4，5 和 7 的数据集
+示例：
+python3 dataset/lamp_downloader.py --tasks 4 5 7
+"""
+
+import os
+import json
+import requests
+from pathlib import Path
+from typing import List, Tuple
+from tqdm import tqdm
+
+
+class LaMPDownloader:
+    """LaMP 数据集下载器"""
+    
+    BASE_URL = "https://ciir.cs.umass.edu/downloads/LaMP"
+    TIME_BASE_URL = "https://ciir.cs.umass.edu/downloads/LaMP/time"
+    
+    # 任务配置（增加了 task 4）
+    TASKS = {
+        4: "LaMP_4",
+        5: "LaMP_5",
+        7: "LaMP_7"
+    }
+    
+    # 数据集分割
+    SPLITS = ["train", "dev", "test"]
+    
+    def __init__(self, output_dir: str = "./lamp_data/lamp"):
+        """
+        初始化下载器
+        
+        Args:
+            output_dir: 输出目录路径
+        """
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+    
+    def download_file(self, url: str, filepath: Path) -> bool:
+        """
+        下载单个文件
+        
+        Args:
+            url: 文件 URL
+            filepath: 保存路径
+            
+        Returns:
+            是否下载成功
+        """
+        try:
+            response = requests.get(url, stream=True, timeout=30)
+            response.raise_for_status()
+            
+            # 创建父目录
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+            
+            # 下载文件
+            total_size = int(response.headers.get('content-length', 0))
+            with open(filepath, 'wb') as f:
+                if total_size == 0:
+                    f.write(response.content)
+                else:
+                    with tqdm(total=total_size, unit='B', unit_scale=True, desc=filepath.name) as pbar:
+                        for chunk in response.iter_content(chunk_size=8192):
+                            if chunk:
+                                f.write(chunk)
+                                pbar.update(len(chunk))
+            
+            return True
+        except Exception as e:
+            print(f"下载失败 {url}: {e}")
+            return False
+    
+    def get_file_urls(self, task_num: int, split: str, setting: str = "user") -> Tuple[str, str]:
+        """
+        获取文件 URL
+        
+        Args:
+            task_num: 任务编号 (4, 5 或 7)
+            split: 数据集分割 (train, dev, test)
+            setting: 设置类型 ("user" 或 "time")
+            
+        Returns:
+            (questions_url, outputs_url) 元组，test 分割时 outputs_url 为 None
+        """
+        task_name = self.TASKS[task_num]
+        base_url = self.BASE_URL if setting == "user" else self.TIME_BASE_URL
+        
+        questions_url = f"{base_url}/{task_name}/{split}/{split}_questions.json"
+        outputs_url = None
+        
+        if split != "test":
+            outputs_url = f"{base_url}/{task_name}/{split}/{split}_outputs.json"
+        
+        return questions_url, outputs_url
+    
+    def download_task(self, task_num: int, setting: str = "user") -> bool:
+        """
+        下载指定任务的所有数据
+        
+        Args:
+            task_num: 任务编号 (4, 5 或 7)
+            setting: 设置类型 ("user" 或 "time")
+            
+        Returns:
+            是否下载成功
+        """
+        if task_num not in self.TASKS:
+            print(f"错误: 不支持的任务编号 {task_num}")
+            return False
+        
+        if setting not in ["user", "time"]:
+            print(f"错误: 不支持的设置类型 {setting}")
+            return False
+        
+        task_name = self.TASKS[task_num]
+        print(f"\n开始下载 LaMP {task_num} ({setting}-based setting)...")
+        
+        success_count = 0
+        total_count = 0
+        
+        for split in self.SPLITS:
+            print(f"\n下载 {split} 分割...")
+            
+            questions_url, outputs_url = self.get_file_urls(task_num, split, setting)
+            
+            # 构建输出路径
+            output_subdir = self.output_dir / f"LaMP_{task_num}" / setting / split
+            questions_path = output_subdir / f"{split}_questions.json"
+            outputs_path = output_subdir / f"{split}_outputs.json" if outputs_url else None
+            
+            # 下载 questions
+            total_count += 1
+            if self.download_file(questions_url, questions_path):
+                success_count += 1
+                print(f"✓ 已下载: {questions_path}")
+            else:
+                print(f"✗ 下载失败: {questions_url}")
+            
+            # 下载 outputs (如果有)
+            if outputs_url:
+                total_count += 1
+                if self.download_file(outputs_url, outputs_path):
+                    success_count += 1
+                    print(f"✓ 已下载: {outputs_path}")
+                else:
+                    print(f"✗ 下载失败: {outputs_url}")
+        
+        print(f"\n任务 {task_num} 下载完成: {success_count}/{total_count} 个文件成功")
+        return success_count == total_count
+    
+    def download_tasks(self, task_nums: List[int], setting: str = "user") -> bool:
+        """
+        下载多个任务
+        
+        Args:
+            task_nums: 任务编号列表
+            setting: 设置类型 ("user" 或 "time")
+            
+        Returns:
+            是否全部下载成功
+        """
+        all_success = True
+        for task_num in task_nums:
+            if not self.download_task(task_num, setting):
+                all_success = False
+        
+        return all_success
+    
+    def verify_downloads(self, task_num: int, setting: str = "user") -> bool:
+        """
+        验证下载的文件
+        
+        Args:
+            task_num: 任务编号
+            setting: 设置类型
+            
+        Returns:
+            是否验证通过
+        """
+        base_dir = self.output_dir / f"LaMP_{task_num}" / setting
+        
+        print(f"\n验证 LaMP {task_num} ({setting}-based) 下载的文件...")
+        
+        all_valid = True
+        
+        for split in self.SPLITS:
+            split_dir = base_dir / split
+            questions_file = split_dir / f"{split}_questions.json"
+            outputs_file = split_dir / f"{split}_outputs.json"
+            
+            # 检查 questions 文件
+            if not questions_file.exists():
+                print(f"✗ 缺失文件: {questions_file}")
+                all_valid = False
+                continue
+            
+            try:
+                with open(questions_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    print(f"✓ {split}_questions.json: {len(data)} 条记录")
+            except Exception as e:
+                print(f"✗ 无法读取 {questions_file}: {e}")
+                all_valid = False
+            
+            # 检查 outputs 文件 (test 分割没有)
+            if split != "test":
+                if not outputs_file.exists():
+                    print(f"✗ 缺失文件: {outputs_file}")
+                    all_valid = False
+                    continue
+                
+                try:
+                    with open(outputs_file, 'r', encoding='utf-8') as f:
+                        data = json.load(f)
+                        print(f"✓ {split}_outputs.json: {len(data)} 条记录")
+                except Exception as e:
+                    print(f"✗ 无法读取 {outputs_file}: {e}")
+                    all_valid = False
+        
+        return all_valid
+
+
+def main():
+    """主函数"""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="下载 LaMP 数据集")
+    parser.add_argument(
+        "--tasks",
+        type=int,
+        nargs="+",
+        default=[4, 5, 7],   # 默认改为 4 5 7
+        help="要下载的任务编号 (默认: 4 5 7)"
+    )
+    parser.add_argument(
+        "--setting",
+        type=str,
+        choices=["user", "time"],
+        default="user",
+        help="数据集设置类型: user (用户分离) 或 time (时间分离) (默认: user)"
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="./data/lamp",
+        help="输出目录 (默认: ./data/lamp)"
+    )
+    parser.add_argument(
+        "--verify",
+        action="store_true",
+        help="下载后验证文件"
+    )
+    
+    args = parser.parse_args()
+    
+    # 创建下载器
+    downloader = LaMPDownloader(output_dir=args.output_dir)
+    
+    # 下载任务
+    print(f"准备下载 LaMP 任务: {args.tasks}")
+    print(f"设置类型: {args.setting}-based")
+    print(f"输出目录: {args.output_dir}")
+    
+    success = downloader.download_tasks(args.tasks, args.setting)
+    
+    # 验证下载
+    if args.verify:
+        for task_num in args.tasks:
+            downloader.verify_downloads(task_num, args.setting)
+    
+    if success:
+        print("\n✓ 所有文件下载成功!")
+    else:
+        print("\n✗ 部分文件下载失败，请检查错误信息")
+    
+    return 0 if success else 1
+
+
+if __name__ == "__main__":
+    exit(main())

Recommend/dataset/lamp_downloader_123.py

@@ -0,0 +1,313 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+LaMP 数据集下载脚本（Task 1 / 2 / 3 版本）
+
+用于下载 LaMP 任务 1，2 和 3 的数据集（支持 user-based 和 time-based）
+
+示例：
+# 下载 user-based 的 1/2/3
+python3 dataset/lamp_downloader_123.py --tasks 1 2 3 --setting user --output-dir ./data/lamp
+
+# 下载 time-based 的 1/2/3
+python3 dataset/lamp_downloader_123.py --tasks 1 2 3 --setting time --output-dir ./data/lamp
+
+# 下载单个 task，例如只要 LaMP_2
+python3 dataset/lamp_downloader_123.py --tasks 2 --setting user --output-dir ./data/lamp --verify
+"""
+import os
+import json
+import requests
+from pathlib import Path
+from typing import List, Tuple
+from tqdm import tqdm
+
+
+class LaMPDownloader:
+    """LaMP 数据集下载器（支持 Task 1 / 2 / 3）"""
+    
+    BASE_URL = "https://ciir.cs.umass.edu/downloads/LaMP"
+    TIME_BASE_URL = "https://ciir.cs.umass.edu/downloads/LaMP/time"
+  
+    # 任务配置：这里改成 1 / 2 / 3
+    TASKS = {
+        1: "LaMP_1",
+        2: "LaMP_2",
+        3: "LaMP_3",
+    }
+    
+    # 数据集分割
+    SPLITS = ["train", "dev", "test"]
+    
+    def __init__(self, output_dir: str = "./data/lamp"):
+        """
+        初始化下载器
+        
+        Args:
+            output_dir: 输出目录路径
+        """
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+    
+    def download_file(self, url: str, filepath: Path) -> bool:
+        """
+        下载单个文件
+        
+        Args:
+            url: 文件 URL
+            filepath: 保存路径
+            
+        Returns:
+            是否下载成功
+        """
+        try:
+            response = requests.get(url, stream=True, timeout=30)
+            response.raise_for_status()
+            
+            # 创建父目录
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+            
+            # 下载文件
+            total_size = int(response.headers.get('content-length', 0))
+            with open(filepath, 'wb') as f:
+                if total_size == 0:
+                    f.write(response.content)
+                else:
+                    with tqdm(total=total_size, unit='B', unit_scale=True, desc=filepath.name) as pbar:
+                        for chunk in response.iter_content(chunk_size=8192):
+                            if chunk:
+                                f.write(chunk)
+                                pbar.update(len(chunk))
+            
+            return True
+        except Exception as e:
+            print(f"下载失败 {url}: {e}")
+            return False
+    
+    def get_file_urls(self, task_num: int, split: str, setting: str = "user") -> Tuple[str, str]:
+        """
+        获取文件 URL
+
+        特殊规则：
+        - LaMP_2 + user-based: train / dev 用 new/ 下的新版数据
+          train: .../LaMP_2/new/train/train_questions.json / train_outputs.json
+          dev : .../LaMP_2/new/dev/dev_questions.json   / dev_outputs.json
+          test: .../LaMP_2/new/test/test_questions.json （无 outputs）
+        """
+        task_name = self.TASKS[task_num]
+        base_url = self.BASE_URL if setting == "user" else self.TIME_BASE_URL
+
+        # ================================
+        # 特殊处理：LaMP_2 + user-based
+        # ================================
+        if task_num == 2 and setting == "user":
+            # train / dev 用 new 目录
+            if split in ["train", "dev"]:
+                prefix = f"{base_url}/{task_name}/new/{split}/{split}"
+                questions_url = f"{prefix}_questions.json"
+                outputs_url = f"{prefix}_outputs.json"
+                return questions_url, outputs_url
+
+            # test 也有 new 版本（只有 questions）
+            if split == "test":
+                questions_url = f"{base_url}/{task_name}/new/test/test_questions.json"
+                return questions_url, None
+
+        # ================================
+        # 默认路径（非 LaMP_2 或非 user-based）
+        # ================================
+        questions_url = f"{base_url}/{task_name}/{split}/{split}_questions.json"
+        outputs_url = None
+
+        # test split 没有 outputs
+        if split != "test":
+            outputs_url = f"{base_url}/{task_name}/{split}/{split}_outputs.json"
+
+        return questions_url, outputs_url
+
+
+    
+    def download_task(self, task_num: int, setting: str = "user") -> bool:
+        """
+        下载指定任务的所有数据
+        
+        Args:
+            task_num: 任务编号 (1, 2 或 3)
+            setting: 设置类型 ("user" 或 "time")
+            
+        Returns:
+            是否下载成功
+        """
+        if task_num not in self.TASKS:
+            print(f"错误: 不支持的任务编号 {task_num}")
+            return False
+        
+        if setting not in ["user", "time"]:
+            print(f"错误: 不支持的设置类型 {setting}")
+            return False
+        
+        print(f"\n开始下载 LaMP {task_num} ({setting}-based setting)...")
+        
+        success_count = 0
+        total_count = 0
+        
+        for split in self.SPLITS:
+            print(f"\n下载 {split} 分割...")
+            
+            questions_url, outputs_url = self.get_file_urls(task_num, split, setting)
+            
+            # 构建输出路径：data/lamp/LaMP_1/user/train/...
+            output_subdir = self.output_dir / f"LaMP_{task_num}" / setting / split
+            questions_path = output_subdir / f"{split}_questions.json"
+            outputs_path = output_subdir / f"{split}_outputs.json" if outputs_url else None
+            
+            # 下载 questions
+            total_count += 1
+            if self.download_file(questions_url, questions_path):
+                success_count += 1
+                print(f"✓ 已下载: {questions_path}")
+            else:
+                print(f"✗ 下载失败: {questions_url}")
+            
+            # 下载 outputs (如果有)
+            if outputs_url:
+                total_count += 1
+                if self.download_file(outputs_url, outputs_path):
+                    success_count += 1
+                    print(f"✓ 已下载: {outputs_path}")
+                else:
+                    print(f"✗ 下载失败: {outputs_url}")
+        
+        print(f"\n任务 {task_num} 下载完成: {success_count}/{total_count} 个文件成功")
+        return success_count == total_count
+    
+    def download_tasks(self, task_nums: List[int], setting: str = "user") -> bool:
+        """
+        下载多个任务
+        
+        Args:
+            task_nums: 任务编号列表
+            setting: 设置类型 ("user" 或 "time")
+            
+        Returns:
+            是否全部下载成功
+        """
+        all_success = True
+        for task_num in task_nums:
+            if not self.download_task(task_num, setting):
+                all_success = False
+        
+        return all_success
+    
+    def verify_downloads(self, task_num: int, setting: str = "user") -> bool:
+        """
+        验证下载的文件
+        
+        Args:
+            task_num: 任务编号
+            setting: 设置类型
+            
+        Returns:
+            是否验证通过
+        """
+        base_dir = self.output_dir / f"LaMP_{task_num}" / setting
+        
+        print(f"\n验证 LaMP {task_num} ({setting}-based) 下载的文件...")
+        
+        all_valid = True
+        
+        for split in self.SPLITS:
+            split_dir = base_dir / split
+            questions_file = split_dir / f"{split}_questions.json"
+            outputs_file = split_dir / f"{split}_outputs.json"
+            
+            # 检查 questions 文件
+            if not questions_file.exists():
+                print(f"✗ 缺失文件: {questions_file}")
+                all_valid = False
+                continue
+            
+            try:
+                with open(questions_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    print(f"✓ {split}_questions.json: {len(data)} 条记录")
+            except Exception as e:
+                print(f"✗ 无法读取 {questions_file}: {e}")
+                all_valid = False
+            
+            # 检查 outputs 文件 (test 分割没有)
+            if split != "test":
+                if not outputs_file.exists():
+                    print(f"✗ 缺失文件: {outputs_file}")
+                    all_valid = False
+                    continue
+                
+                try:
+                    with open(outputs_file, 'r', encoding='utf-8') as f:
+                        data = json.load(f)
+                        print(f"✓ {split}_outputs.json: {len(data)} 条记录")
+                except Exception as e:
+                    print(f"✗ 无法读取 {outputs_file}: {e}")
+                    all_valid = False
+        
+        return all_valid
+
+
+def main():
+    """主函数"""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="下载 LaMP 数据集 (Tasks 1 / 2 / 3)")
+    parser.add_argument(
+        "--tasks",
+        type=int,
+        nargs="+",
+        default=[1, 2, 3],   # 默认改为 1 2 3
+        help="要下载的任务编号 (默认: 1 2 3)"
+    )
+    parser.add_argument(
+        "--setting",
+        type=str,
+        choices=["user", "time"],
+        default="user",
+        help="数据集设置类型: user (用户分离) 或 time (时间分离) (默认: user)"
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="./data/lamp",
+        help="输出目录 (默认: ./data/lamp)"
+    )
+    parser.add_argument(
+        "--verify",
+        action="store_true",
+        help="下载后验证文件"
+    )
+    
+    args = parser.parse_args()
+    
+    # 创建下载器
+    downloader = LaMPDownloader(output_dir=args.output_dir)
+    
+    # 下载任务
+    print(f"准备下载 LaMP 任务: {args.tasks}")
+    print(f"设置类型: {args.setting}-based")
+    print(f"输出目录: {args.output_dir}")
+    
+    success = downloader.download_tasks(args.tasks, args.setting)
+    
+    # 验证下载
+    if args.verify:
+        for task_num in args.tasks:
+            downloader.verify_downloads(task_num, args.setting)
+    
+    if success:
+        print("\n✓ 所有文件下载成功!")
+    else:
+        print("\n✗ 部分文件下载失败，请检查错误信息")
+    
+    return 0 if success else 1
+
+
+if __name__ == "__main__":
+    exit(main())

Recommend/dataset/processing.py

@@ -0,0 +1,56 @@
+'''
+把原始数据集分成一个小样本数据集，用于快速调试
+'''
+
+
+
+import json
+import random
+import os
+
+def process_data(input_path, output_path, sample_ratio=0.1):
+    """处理数据并只保存指定比例的数据"""
+    print(f"处理文件: {input_path}")
+    
+    # 读取原始数据
+    with open(input_path, 'r') as f:
+        data = [json.loads(line) for line in f if line.strip()]
+    
+    print(f"原始数据量: {len(data)} 条")
+    
+    # 随机采样10%的数据
+    sample_size = int(len(data) * sample_ratio)
+    sampled_data = random.sample(data, sample_size)
+    
+    print(f"采样数据量: {len(sampled_data)} 条 ({sample_ratio*100}%)")
+    
+    # 保存处理后的数据
+    with open(output_path, 'w') as f:
+        json.dump(sampled_data, f, ensure_ascii=False, indent=2)
+    
+    print(f"数据已保存到: {output_path}")
+    print("-" * 50)
+
+# 设置随机种子确保结果可重现
+random.seed(42)
+
+# 创建processed_data文件夹
+processed_data_dir = '/opt/tiger/PLLM1.0/dataset/processed_data'
+if not os.path.exists(processed_data_dir):
+    os.makedirs(processed_data_dir)
+    print(f"✅ 创建文件夹: {processed_data_dir}")
+else:
+    print(f"📁 文件夹已存在: {processed_data_dir}")
+
+# 处理训练数据
+train_input_path = '/opt/tiger/PLLM1.0/dataset/abstract_train.json'
+train_output_path = '/opt/tiger/PLLM1.0/dataset/processed_data/abstract_train.json'
+process_data(train_input_path, train_output_path, sample_ratio=0.1)
+
+# 处理测试数据
+test_input_path = '/opt/tiger/PLLM1.0/dataset/abstract_test.json'
+test_output_path = '/opt/tiger/PLLM1.0/dataset/processed_data/abstract_test.json'
+process_data(test_input_path, test_output_path, sample_ratio=0.1)
+
+print("✅ 数据处理完成！")
+