Download scripts/download_tfs_docs.py from aionaion/chatdoc: direct link, hf CLI and curl.
- Browser
- Download file 7.63 kB
-
https://huggingface.co/spaces/aionaion/chatdoc/resolve/main/scripts/download_tfs_docs.py
- Command line
-
hf download hf://spaces/aionaion/chatdoc/scripts/download_tfs_docs.py
-
curl -L -o download_tfs_docs.py https://huggingface.co/spaces/aionaion/chatdoc/resolve/main/scripts/download_tfs_docs.py
7.63 kB
| #!/usr/bin/env python3 | |
| """ | |
| Script to download documents from specified TFS repositories. | |
| Uses the same filtering logic as list_tfs_docs.py. | |
| """ | |
| import sys | |
| import argparse | |
| from pathlib import Path | |
| import logging | |
| import re | |
| from typing import List, Dict, Tuple | |
| # Add the app directory to the path | |
| sys.path.append(str(Path(__file__).parent.parent)) | |
| from app.utils.tfs_client import TFSClient | |
| from app.core.config import settings | |
| # --- LISTA DE REPOSITÓRIOS PADRÃO --- | |
| DEFAULT_REPO_PATTERNS = [ | |
| "cpcstools", | |
| "PEX", | |
| 'ACEPC' | |
| 'ACIA_SIGS_DOC', | |
| 'ACEITE_EPCCPT_DOC', | |
| 'ACIA_SIGS_DOC', | |
| 'COLETOR_AppColetor_DOC', | |
| 'GERENTE_IMSIP_DOC', | |
| 'INTEGRACOES_CPCSSERVICES_DOC', | |
| ] | |
| # --- LISTA DE ARQUIVOS E PASTAS EXCLUÍDOS (Case Insensitive) --- | |
| EXCLUDED_NAMES = [ | |
| "lib_list.txt", | |
| "pull_request_template.md", | |
| "contributing.md", | |
| ".azuredevops" | |
| ] | |
| # --- PASTAS PERMITIDAS PARA BUSCA (além da raiz) --- | |
| ALLOWED_DOC_FOLDERS = [ | |
| "doc", "docs", | |
| "documentacao", "documentação", | |
| "manual", "manuais", | |
| "config", "configs", | |
| # "spec", "specs" | |
| ] | |
| # ------------------------------------ | |
| def is_path_allowed(file_path_str: str) -> bool: | |
| """ | |
| Verifica se o arquivo deve ser incluído baseado na sua localização. | |
| Regra: Aceita se estiver na raiz OU se estiver dentro de uma pasta permitida de PRIMEIRO NÍVEL. | |
| """ | |
| path_str = file_path_str.replace('\\', '/') | |
| if path_str.startswith('/'): | |
| path_str = path_str[1:] | |
| path_obj = Path(path_str) | |
| parts = path_obj.parts | |
| if len(parts) == 1: | |
| return True | |
| first_folder = parts[0] | |
| if first_folder.lower() in ALLOWED_DOC_FOLDERS: | |
| return True | |
| return False | |
| def filter_latest_versions(file_paths: List[str]) -> List[str]: | |
| """ | |
| Filtra arquivos para manter apenas a versão mais recente. | |
| Trata 'Arquivo.ext' como versão 0 e 'Arquivo_V1.ext' como versão 1. | |
| """ | |
| version_pattern = re.compile(r'^(.*?)[\s_]*[vV](\d+)$') | |
| versioned_files: Dict[str, Tuple[int, str]] = {} | |
| for file_path in file_paths: | |
| path_obj = Path(file_path) | |
| stem = path_obj.stem | |
| match = version_pattern.match(stem) | |
| if match: | |
| base_name = match.group(1).strip() | |
| version = int(match.group(2)) | |
| else: | |
| base_name = stem.strip() | |
| version = 0 | |
| key = f"{path_obj.parent}/{base_name}{path_obj.suffix}".lower() | |
| if key not in versioned_files or version > versioned_files[key][0]: | |
| versioned_files[key] = (version, file_path) | |
| return sorted([info[1] for info in versioned_files.values()]) | |
| def main(): | |
| """Main function to download documents""" | |
| parser = argparse.ArgumentParser(description="Download documents from TFS repositories") | |
| parser.add_argument( | |
| "patterns", | |
| nargs='*', | |
| default=None, | |
| help="Optional: List of repository name patterns to search for" | |
| ) | |
| parser.add_argument( | |
| "--dry-run", | |
| action="store_true", | |
| help="Simulate download without saving files" | |
| ) | |
| parser.add_argument( | |
| "--verbose", "-v", | |
| action="store_true", | |
| help="Enable verbose logging" | |
| ) | |
| args = parser.parse_args() | |
| if args.patterns: | |
| repo_patterns_to_use = args.patterns | |
| print("🔍 Using repository patterns from command line.") | |
| else: | |
| repo_patterns_to_use = DEFAULT_REPO_PATTERNS | |
| print("🔍 Using predefined repository patterns from the script.") | |
| log_level = logging.DEBUG if args.verbose else logging.INFO | |
| logging.basicConfig( | |
| level=log_level, | |
| format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' | |
| ) | |
| logger = logging.getLogger(__name__) | |
| if not all([settings.tfs_url, settings.tfs_pat, settings.tfs_project]): | |
| print("❌ TFS not configured. Please set TFS_URL, TFS_PAT, and TFS_PROJECT in your .env file.") | |
| return | |
| # Resolve o diretório de destino de forma absoluta | |
| raw_path_setting = Path(settings.documents_directory_raw) | |
| if not raw_path_setting.is_absolute(): | |
| # Se for relativo, ancora na raiz do projeto (2 níveis acima deste script) | |
| project_root = Path(__file__).resolve().parent.parent | |
| target_dir = (project_root / raw_path_setting).resolve(strict=False) | |
| else: | |
| target_dir = raw_path_setting.resolve(strict=False) | |
| print("=" * 60) | |
| print(" TFS Document Downloader (Smart Filter v5)") | |
| print(f" Target Directory: {target_dir}") | |
| print("=" * 60) | |
| try: | |
| tfs_client = TFSClient() | |
| if not tfs_client.connection: | |
| print("❌ Failed to connect to TFS.") | |
| return | |
| supported_extensions = settings.supported_extensions | |
| print(f"🔍 Searching for files with extensions: {', '.join(supported_extensions)}") | |
| print(f"📂 Allowed root folders: {', '.join(ALLOWED_DOC_FOLDERS)}") | |
| print(f"🚫 Excluding names: {', '.join(EXCLUDED_NAMES)}\n") | |
| total_downloaded = 0 | |
| for pattern in repo_patterns_to_use: | |
| print(f"--- Searching for repositories matching pattern: '{pattern}' ---") | |
| matching_repos = tfs_client.get_repositories(name_pattern=pattern) | |
| if not matching_repos: | |
| print(f" No repositories found matching '{pattern}'.") | |
| continue | |
| for repo in matching_repos: | |
| print(f"\n Processing repository: {repo.name}") | |
| files = tfs_client.list_files_in_repo(repo.name, supported_extensions) | |
| non_excluded_files = [ | |
| f for f in files | |
| if Path(f).name.lower() not in [e.lower() for e in EXCLUDED_NAMES] and | |
| not any(part.lower() in [e.lower() for e in EXCLUDED_NAMES] for part in Path(f).parts) | |
| ] | |
| location_filtered_files = [f for f in non_excluded_files if is_path_allowed(f)] | |
| final_files = filter_latest_versions(location_filtered_files) | |
| if final_files: | |
| print(f" Found {len(final_files)} documents to download:") | |
| for f in final_files: | |
| print(f" - {f}") | |
| if not args.dry_run: | |
| print(" ⬇️ Downloading...") | |
| count = tfs_client.download_documents(repo.name, final_files, target_dir) | |
| total_downloaded += count | |
| print(f" ✅ Downloaded {count} files.") | |
| else: | |
| print(" [DRY RUN] Skipping download.") | |
| else: | |
| print(" No documents to download after filtering.") | |
| print("-" * (len(pattern) + 40) + "\n") | |
| print("=" * 60) | |
| print(f"🎉 Total files downloaded: {total_downloaded}") | |
| print(f"📂 Files saved to: {target_dir}") | |
| print("=" * 60) | |
| except Exception as e: | |
| logger.error(f"An unexpected error occurred: {e}", exc_info=True) | |
| print(f"❌ An unexpected error occurred. Check the logs for details.") | |
| if __name__ == "__main__": | |
| main() | |