Spaces:
Sleeping
Sleeping
Download create_and_upload_index.py from MTORXYZ/MTOR-Testing: direct link, hf CLI and curl.
- Browser
- Download file 4.07 kB
-
https://huggingface.co/spaces/MTORXYZ/MTOR-Testing/resolve/main/create_and_upload_index.py
- Command line
-
hf download hf://spaces/MTORXYZ/MTOR-Testing/create_and_upload_index.py
-
curl -L -o create_and_upload_index.py https://huggingface.co/spaces/MTORXYZ/MTOR-Testing/resolve/main/create_and_upload_index.py
4.07 kB
| import os | |
| import json | |
| from tqdm import tqdm | |
| from azure.core.credentials import AzureKeyCredential | |
| from azure.search.documents.indexes import SearchIndexClient | |
| from azure.search.documents.indexes.models import ( | |
| SearchIndex, | |
| SearchField, | |
| SearchFieldDataType, | |
| SimpleField, | |
| SearchableField, | |
| VectorSearch, | |
| HnswAlgorithmConfiguration, | |
| VectorSearchProfile | |
| ) | |
| from azure.search.documents import SearchClient | |
| from openai import AzureOpenAI | |
| from dotenv import load_dotenv | |
| load_dotenv() | |
| # ---------------- CONFIG ---------------- | |
| AZURE_SEARCH_ENDPOINT = os.getenv("AZURE_SEARCH_ENDPOINT") # e.g., https://your-search.search.windows.net | |
| AZURE_SEARCH_KEY = os.getenv("AZURE_SEARCH_KEY") | |
| AZURE_SEARCH_INDEX_NAME = "it-ticket-solutions-index" | |
| HF_TOKEN = os.getenv("HF_TOKEN") | |
| API_BASE_URL = os.getenv("API_BASE_URL") | |
| MODEL_NAME = os.getenv("MODEL_NAME") # e.g., | |
| AZURE_OPENAI_API_VERSION = "2024-02-15-preview" | |
| DATA_FILE = "data/knowledge_base.json" | |
| VECTOR_DIMENSIONS = 1536 # Based on OpenAI embeddings | |
| # ---------------------------------------- | |
| # ---------- Clients ---------- | |
| credential = AzureKeyCredential(AZURE_SEARCH_KEY) | |
| index_client = SearchIndexClient(endpoint=AZURE_SEARCH_ENDPOINT, credential=credential) | |
| search_client = SearchClient(endpoint=AZURE_SEARCH_ENDPOINT, index_name=AZURE_SEARCH_INDEX_NAME, credential=credential) | |
| openai_client = AzureOpenAI(api_key=HF_TOKEN, api_version=AZURE_OPENAI_API_VERSION, azure_endpoint=API_BASE_URL) | |
| # ---------- Index Creation ---------- | |
| def create_index(): | |
| print("📦 Creating index...") | |
| fields = [ | |
| SimpleField(name="id", type=SearchFieldDataType.String, key=True, filterable=True, sortable=True, facetable=True), | |
| SearchableField(name="category", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True), | |
| SearchableField(name="problem", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True), | |
| SearchableField(name="solution", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True), | |
| SearchField( | |
| name="embedding", | |
| type=SearchFieldDataType.Collection(SearchFieldDataType.Single), | |
| searchable=True, | |
| vector_search_dimensions=VECTOR_DIMENSIONS, | |
| vector_search_profile_name="default" | |
| ) | |
| ] | |
| vector_search = VectorSearch( | |
| algorithms=[HnswAlgorithmConfiguration(name="default")], | |
| profiles=[VectorSearchProfile(name="default", algorithm_configuration_name="default")] | |
| ) | |
| index = SearchIndex(name=AZURE_SEARCH_INDEX_NAME, fields=fields, vector_search=vector_search) | |
| try: | |
| # Try to get the index; if it exists, we skip creation | |
| index_client.get_index(name=AZURE_SEARCH_INDEX_NAME) | |
| print(f"ℹ️ Index '{AZURE_SEARCH_INDEX_NAME}' already exists. Skipping creation.") | |
| except Exception: | |
| # If not found, create the index | |
| index_client.create_index(index) | |
| print(f"✅ Index '{AZURE_SEARCH_INDEX_NAME}' created successfully.") | |
| # ---------- Embedding ---------- | |
| def embed_text(text: str): | |
| response = openai_client.embeddings.create( | |
| input=[text], | |
| model=MODEL_NAME | |
| ) | |
| return response.data[0].embedding | |
| # ---------- Upload ---------- | |
| def load_data(): | |
| with open(DATA_FILE, "r", encoding="utf-8") as f: | |
| return json.load(f) | |
| def upload_documents(docs): | |
| print(f"\n📤 Uploading {len(docs)} documents...") | |
| batch_size = 10 | |
| for i in tqdm(range(0, len(docs), batch_size)): | |
| chunk = docs[i:i+batch_size] | |
| search_client.upload_documents(documents=chunk) | |
| print("✅ Upload completed.") | |
| # ---------- Main ---------- | |
| def main(): | |
| create_index() | |
| raw_docs = load_data() | |
| enriched_docs = [] | |
| print("🔍 Generating embeddings...") | |
| for doc in tqdm(raw_docs): | |
| embedding = embed_text(doc["problem"]) | |
| doc["embedding"] = embedding | |
| enriched_docs.append(doc) | |
| upload_documents(enriched_docs) | |
| if __name__ == "__main__": | |
| main() |