File size: 4,068 Bytes
7ed7979
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f72327a
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

import os
import json
from tqdm import tqdm
from azure.core.credentials import AzureKeyCredential
from azure.search.documents.indexes import SearchIndexClient
from azure.search.documents.indexes.models import (
    SearchIndex,
    SearchField,
    SearchFieldDataType,
    SimpleField,
    SearchableField,
    VectorSearch,
    HnswAlgorithmConfiguration,
    VectorSearchProfile
)
from azure.search.documents import SearchClient
from openai import AzureOpenAI
from dotenv import load_dotenv

load_dotenv()

# ---------------- CONFIG ----------------
AZURE_SEARCH_ENDPOINT = os.getenv("AZURE_SEARCH_ENDPOINT")  # e.g., https://your-search.search.windows.net
AZURE_SEARCH_KEY = os.getenv("AZURE_SEARCH_KEY")
AZURE_SEARCH_INDEX_NAME = "it-ticket-solutions-index"

HF_TOKEN = os.getenv("HF_TOKEN")
API_BASE_URL = os.getenv("API_BASE_URL")
MODEL_NAME = os.getenv("MODEL_NAME")  # e.g.,
AZURE_OPENAI_API_VERSION = "2024-02-15-preview"

DATA_FILE = "data/knowledge_base.json"
VECTOR_DIMENSIONS = 1536  # Based on OpenAI embeddings
# ----------------------------------------

# ---------- Clients ----------
credential = AzureKeyCredential(AZURE_SEARCH_KEY)
index_client = SearchIndexClient(endpoint=AZURE_SEARCH_ENDPOINT, credential=credential)
search_client = SearchClient(endpoint=AZURE_SEARCH_ENDPOINT, index_name=AZURE_SEARCH_INDEX_NAME, credential=credential)
openai_client = AzureOpenAI(api_key=HF_TOKEN, api_version=AZURE_OPENAI_API_VERSION, azure_endpoint=API_BASE_URL)


# ---------- Index Creation ----------
def create_index():
    print("📦 Creating index...")

    fields = [
        SimpleField(name="id", type=SearchFieldDataType.String, key=True, filterable=True, sortable=True, facetable=True),
        SearchableField(name="category", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True),
        SearchableField(name="problem", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True),
        SearchableField(name="solution", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True),
        SearchField(
            name="embedding",
            type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
            searchable=True,
            vector_search_dimensions=VECTOR_DIMENSIONS,
            vector_search_profile_name="default"
        )
    ]

    vector_search = VectorSearch(
        algorithms=[HnswAlgorithmConfiguration(name="default")],
        profiles=[VectorSearchProfile(name="default", algorithm_configuration_name="default")]
    )

    index = SearchIndex(name=AZURE_SEARCH_INDEX_NAME, fields=fields, vector_search=vector_search)

    try:
        # Try to get the index; if it exists, we skip creation
        index_client.get_index(name=AZURE_SEARCH_INDEX_NAME)
        print(f"ℹ️ Index '{AZURE_SEARCH_INDEX_NAME}' already exists. Skipping creation.")
    except Exception:
        # If not found, create the index
        index_client.create_index(index)
        print(f"✅ Index '{AZURE_SEARCH_INDEX_NAME}' created successfully.")


# ---------- Embedding ----------
def embed_text(text: str):
    response = openai_client.embeddings.create(
        input=[text],
        model=MODEL_NAME
    )
    return response.data[0].embedding


# ---------- Upload ----------
def load_data():
    with open(DATA_FILE, "r", encoding="utf-8") as f:
        return json.load(f)


def upload_documents(docs):
    print(f"\n📤 Uploading {len(docs)} documents...")
    batch_size = 10
    for i in tqdm(range(0, len(docs), batch_size)):
        chunk = docs[i:i+batch_size]
        search_client.upload_documents(documents=chunk)
    print("✅ Upload completed.")

# ---------- Main ----------
def main():
    create_index()

    raw_docs = load_data()
    enriched_docs = []

    print("🔍 Generating embeddings...")
    for doc in tqdm(raw_docs):
        embedding = embed_text(doc["problem"])
        doc["embedding"] = embedding
        enriched_docs.append(doc)

    upload_documents(enriched_docs)


if __name__ == "__main__":
    main()