TANMIHA commited on
Commit
7ed7979
·
verified ·
1 Parent(s): 548dfe3

Update create_and_upload_index.py

Browse files
Files changed (1) hide show
  1. create_and_upload_index.py +118 -118
create_and_upload_index.py CHANGED
@@ -1,119 +1,119 @@
1
-
2
- import os
3
- import json
4
- from tqdm import tqdm
5
- from azure.core.credentials import AzureKeyCredential
6
- from azure.search.documents.indexes import SearchIndexClient
7
- from azure.search.documents.indexes.models import (
8
- SearchIndex,
9
- SearchField,
10
- SearchFieldDataType,
11
- SimpleField,
12
- SearchableField,
13
- VectorSearch,
14
- HnswAlgorithmConfiguration,
15
- VectorSearchProfile
16
- )
17
- from azure.search.documents import SearchClient
18
- from openai import AzureOpenAI
19
- from dotenv import load_dotenv
20
-
21
- load_dotenv()
22
-
23
- # ---------------- CONFIG ----------------
24
- AZURE_SEARCH_ENDPOINT = os.getenv("AZURE_SEARCH_ENDPOINT") # e.g., https://your-search.search.windows.net
25
- AZURE_SEARCH_KEY = os.getenv("AZURE_SEARCH_KEY")
26
- AZURE_SEARCH_INDEX_NAME = "it-ticket-solutions-index"
27
-
28
- HF_TOKEN = os.getenv("HF_TOKEN")
29
- API_BASE_URL = os.getenv("API_BASE_URL")
30
- MODEL_NAME = os.getenv("MODEL_NAME") # e.g.,
31
- AZURE_OPENAI_API_VERSION = "2024-02-15-preview"
32
-
33
- DATA_FILE = "data/knowledge_base.json"
34
- VECTOR_DIMENSIONS = 1536 # Based on OpenAI embeddings
35
- # ----------------------------------------
36
-
37
- # ---------- Clients ----------
38
- credential = AzureKeyCredential(AZURE_SEARCH_KEY)
39
- index_client = SearchIndexClient(endpoint=AZURE_SEARCH_ENDPOINT, credential=credential)
40
- search_client = SearchClient(endpoint=AZURE_SEARCH_ENDPOINT, index_name=AZURE_SEARCH_INDEX_NAME, credential=credential)
41
- openai_client = AzureOpenAI(api_key=HF_TOKEN, api_version=AZURE_OPENAI_API_VERSION, azure_endpoint=API_BASE_URL)
42
-
43
-
44
- # ---------- Index Creation ----------
45
- def create_index():
46
- print("📦 Creating index...")
47
-
48
- fields = [
49
- SimpleField(name="id", type=SearchFieldDataType.String, key=True, filterable=True, sortable=True, facetable=True),
50
- SearchableField(name="category", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True),
51
- SearchableField(name="problem", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True),
52
- SearchableField(name="solution", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True),
53
- SearchField(
54
- name="embedding",
55
- type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
56
- searchable=True,
57
- vector_search_dimensions=VECTOR_DIMENSIONS,
58
- vector_search_profile_name="default"
59
- )
60
- ]
61
-
62
- vector_search = VectorSearch(
63
- algorithms=[HnswAlgorithmConfiguration(name="default")],
64
- profiles=[VectorSearchProfile(name="default", algorithm_configuration_name="default")]
65
- )
66
-
67
- index = SearchIndex(name=AZURE_SEARCH_INDEX_NAME, fields=fields, vector_search=vector_search)
68
-
69
- try:
70
- # Try to get the index; if it exists, we skip creation
71
- index_client.get_index(name=AZURE_SEARCH_INDEX_NAME)
72
- print(f"ℹ️ Index '{AZURE_SEARCH_INDEX_NAME}' already exists. Skipping creation.")
73
- except Exception:
74
- # If not found, create the index
75
- index_client.create_index(index)
76
- print(f"✅ Index '{AZURE_SEARCH_INDEX_NAME}' created successfully.")
77
-
78
-
79
- # ---------- Embedding ----------
80
- def embed_text(text: str):
81
- response = openai_client.embeddings.create(
82
- input=[text],
83
- model=MODEL_NAME
84
- )
85
- return response.data[0].embedding
86
-
87
-
88
- # ---------- Upload ----------
89
- def load_data():
90
- with open(DATA_FILE, "r", encoding="utf-8") as f:
91
- return json.load(f)
92
-
93
-
94
- def upload_documents(docs):
95
- print(f"\n📤 Uploading {len(docs)} documents...")
96
- batch_size = 10
97
- for i in tqdm(range(0, len(docs), batch_size)):
98
- chunk = docs[i:i+batch_size]
99
- search_client.upload_documents(documents=chunk)
100
- print("✅ Upload completed.")
101
-
102
- # ---------- Main ----------
103
- def main():
104
- create_index()
105
-
106
- raw_docs = load_data()
107
- enriched_docs = []
108
-
109
- print("🔍 Generating embeddings...")
110
- for doc in tqdm(raw_docs):
111
- embedding = embed_text(doc["problem"])
112
- doc["embedding"] = embedding
113
- enriched_docs.append(doc)
114
-
115
- upload_documents(enriched_docs)
116
-
117
-
118
- if __name__ == "__main__":
119
  main()
 
1
+
2
+ import os
3
+ import json
4
+ from tqdm import tqdm
5
+ from azure.core.credentials import AzureKeyCredential
6
+ from azure.search.documents.indexes import SearchIndexClient
7
+ from azure.search.documents.indexes.models import (
8
+ SearchIndex,
9
+ SearchField,
10
+ SearchFieldDataType,
11
+ SimpleField,
12
+ SearchableField,
13
+ VectorSearch,
14
+ HnswAlgorithmConfiguration,
15
+ VectorSearchProfile
16
+ )
17
+ from azure.search.documents import SearchClient
18
+ from openai import AzureOpenAI
19
+ from dotenv import load_dotenv
20
+
21
+ load_dotenv()
22
+
23
+ # ---------------- CONFIG ----------------
24
+ AZURE_SEARCH_ENDPOINT = os.getenv("AZURE_SEARCH_ENDPOINT") # e.g., https://your-search.search.windows.net
25
+ AZURE_SEARCH_KEY = os.getenv("AZURE_SEARCH_KEY")
26
+ AZURE_SEARCH_INDEX_NAME = "it-ticket-solutions-index"
27
+
28
+ HF_TOKEN = os.getenv("HF_TOKEN")
29
+ API_BASE_URL = os.getenv("API_BASE_URL")
30
+ MODEL_NAME = os.getenv("MODEL_NAME") # e.g.,
31
+ AZURE_OPENAI_API_VERSION = "2024-02-15-preview"
32
+
33
+ DATA_FILE = "data/knowledge_base.json"
34
+ VECTOR_DIMENSIONS = 1536 # Based on OpenAI embeddings
35
+ # ----------------------------------------
36
+
37
+ # ---------- Clients ----------
38
+ credential = AzureKeyCredential(AZURE_SEARCH_KEY)
39
+ index_client = SearchIndexClient(endpoint=AZURE_SEARCH_ENDPOINT, credential=credential)
40
+ search_client = SearchClient(endpoint=AZURE_SEARCH_ENDPOINT, index_name=AZURE_SEARCH_INDEX_NAME, credential=credential)
41
+ openai_client = AzureOpenAI(api_key=HF_TOKEN, api_version=AZURE_OPENAI_API_VERSION, azure_endpoint=API_BASE_URL)
42
+
43
+
44
+ # ---------- Index Creation ----------
45
+ def create_index():
46
+ print("📦 Creating index...")
47
+
48
+ fields = [
49
+ SimpleField(name="id", type=SearchFieldDataType.String, key=True, filterable=True, sortable=True, facetable=True),
50
+ SearchableField(name="category", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True),
51
+ SearchableField(name="problem", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True),
52
+ SearchableField(name="solution", type=SearchFieldDataType.String, filterable=True, sortable=True, facetable=True),
53
+ SearchField(
54
+ name="embedding",
55
+ type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
56
+ searchable=True,
57
+ vector_search_dimensions=VECTOR_DIMENSIONS,
58
+ vector_search_profile_name="default"
59
+ )
60
+ ]
61
+
62
+ vector_search = VectorSearch(
63
+ algorithms=[HnswAlgorithmConfiguration(name="default")],
64
+ profiles=[VectorSearchProfile(name="default", algorithm_configuration_name="default")]
65
+ )
66
+
67
+ index = SearchIndex(name=AZURE_SEARCH_INDEX_NAME, fields=fields, vector_search=vector_search)
68
+
69
+ try:
70
+ # Try to get the index; if it exists, we skip creation
71
+ index_client.get_index(name=AZURE_SEARCH_INDEX_NAME)
72
+ print(f"ℹ️ Index '{AZURE_SEARCH_INDEX_NAME}' already exists. Skipping creation.")
73
+ except Exception:
74
+ # If not found, create the index
75
+ index_client.create_index(index)
76
+ print(f"✅ Index '{AZURE_SEARCH_INDEX_NAME}' created successfully.")
77
+
78
+
79
+ # ---------- Embedding ----------
80
+ def embed_text(text: str):
81
+ response = openai_client.embeddings.create(
82
+ input=[text],
83
+ model=MODEL_NAME
84
+ )
85
+ return response.data[0].embedding
86
+
87
+
88
+ # ---------- Upload ----------
89
+ def load_data():
90
+ with open(DATA_FILE, "r", encoding="utf-8") as f:
91
+ return json.load(f)
92
+
93
+
94
+ def upload_documents(docs):
95
+ print(f"\n📤 Uploading {len(docs)} documents...")
96
+ batch_size = 10
97
+ for i in tqdm(range(0, len(docs), batch_size)):
98
+ chunk = docs[i:i+batch_size]
99
+ search_client.upload_documents(documents=chunk)
100
+ print("✅ Upload completed.")
101
+
102
+ # ---------- Main ----------
103
+ def main():
104
+ create_index()
105
+
106
+ raw_docs = load_data()
107
+ enriched_docs = []
108
+
109
+ print("🔍 Generating embeddings...")
110
+ for doc in tqdm(raw_docs):
111
+ embedding = embed_text(doc["problem"])
112
+ doc["embedding"] = embedding
113
+ enriched_docs.append(doc)
114
+
115
+ upload_documents(enriched_docs)
116
+
117
+
118
+ if __name__ == "__main__":
119
  main()