File size: 29,437 Bytes
54656fc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
"""
Lazy-initialized singletons for heavy resources (models, DB).
Replaces @st.cache_resource from app.py.
"""
import hashlib
import gc
import json
import logging
import os
import time

logger = logging.getLogger(__name__)

from .config import (
    EMBED_MODEL_PATH, BASE_MODEL_EMBED, EMBEDDER_BACKEND, SIMCSE_POOLING,
    SUMMARY_MODEL_PATH, CLASSIFY_MODEL_PATH,
    INDEX_MANIFEST_PATH, NOTICES_CACHE_PATH,
    VECTOR_DB, PINECONE_API_KEY, PINECONE_INDEX_NAME, PINECONE_CLOUD,
    PINECONE_REGION, PINECONE_NAMESPACE, PINECONE_CACHE_PATH, EMBEDDING_DIM,
    CHUNK_SIZE, CHUNK_OVERLAP, EMBEDDING_DEVICE,
)
from .utils import infer_category, chunk_text

_embed_model      = None
_summary_pipeline = None
_classifier       = None
_label_map: dict  = {}
_vector_collection = None

EMBEDDING_PIPELINE_VERSION = (
    f"simcse-{SIMCSE_POOLING}-v1"
    if EMBEDDER_BACKEND == "simcse"
    else "sentence-transformers-default-v1"
)
TEXT_PROCESSING_VERSION = "ko-compound-category-v2"


class SimCSEEmbedder:
    """
    SimCSE-aware embedder with selectable pooling.

    CLS pooling follows the common SimCSE inference path. Mean pooling is useful
    as an ablation for longer retrieval chunks where token-level evidence matters.
    """

    def __init__(self, model_path: str, device: str = "cpu", pooling: str = SIMCSE_POOLING) -> None:
        from transformers import AutoTokenizer, AutoModel

        pooling = pooling.lower()
        if pooling not in {"cls", "mean"}:
            raise ValueError(f"Unsupported pooling: {pooling}. Use 'cls' or 'mean'.")

        local_only = os.getenv("TRANSFORMERS_OFFLINE") == "1" or os.getenv("HF_HUB_OFFLINE") == "1"
        self._tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=local_only)
        self._model = AutoModel.from_pretrained(model_path, local_files_only=local_only)
        self._model.eval()
        self._device = device
        self._pooling = pooling
        self._model.to(device)

    def encode(
        self,
        sentences: "str | list[str]",
        batch_size: int = 64,
        show_progress_bar: bool = False,
    ) -> "np.ndarray":
        import numpy as np
        import torch
        import torch.nn.functional as F

        single = isinstance(sentences, str)
        if single:
            sentences = [sentences]

        all_embeddings: list = []
        total = len(sentences)
        total_batches = (total + batch_size - 1) // batch_size
        started_at = time.monotonic()
        for start in range(0, len(sentences), batch_size):
            batch = sentences[start : start + batch_size]
            batch_no = start // batch_size + 1
            if show_progress_bar:
                logger.info(
                    "청크 인코딩 진행 중: %d/%d 배치 (%d/%d 청크)",
                    batch_no,
                    total_batches,
                    min(start + len(batch), total),
                    total,
                )
            encoded = self._tokenizer(
                batch,
                padding=True,
                truncation=True,
                max_length=512,
                return_tensors="pt",
            ).to(self._device)
            with torch.inference_mode():
                output = self._model(**encoded)
            hidden = output.last_hidden_state
            if self._pooling == "cls":
                pooled = hidden[:, 0, :]
            else:
                mask = encoded["attention_mask"].unsqueeze(-1).to(hidden.dtype)
                pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-9)
            pooled = F.normalize(pooled, p=2, dim=1)
            all_embeddings.append(pooled.cpu().numpy())
            if show_progress_bar:
                elapsed = time.monotonic() - started_at
                logger.info(
                    "청크 인코딩 완료: %d/%d 배치 (%.1f%%, %.1fs 경과)",
                    batch_no,
                    total_batches,
                    batch_no / total_batches * 100,
                    elapsed,
                )

        result = np.concatenate(all_embeddings, axis=0)
        return result[0] if single else result


def _release_torch_cache() -> None:
    gc.collect()
    try:
        import torch
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
        if torch.backends.mps.is_available():
            torch.mps.empty_cache()
    except Exception:
        logger.debug("Torch cache release skipped", exc_info=True)


def _best_device() -> str:
    import torch
    if EMBEDDING_DEVICE:
        return EMBEDDING_DEVICE
    if torch.backends.mps.is_available():
        return "mps"
    if torch.cuda.is_available():
        return "cuda"
    return "cpu"


def get_embed_model():
    global _embed_model
    if _embed_model is None:
        if EMBEDDER_BACKEND == "simcse":
            _embed_model = SimCSEEmbedder(_embed_model_source(), device=_best_device())
        elif EMBEDDER_BACKEND in {"sentence-transformers", "sentence_transformers", "st"}:
            from sentence_transformers import SentenceTransformer

            local_only = os.getenv("TRANSFORMERS_OFFLINE") == "1" or os.getenv("HF_HUB_OFFLINE") == "1"
            _embed_model = SentenceTransformer(
                _embed_model_source(),
                device=_best_device(),
                local_files_only=local_only,
            )
        else:
            raise ValueError(
                f"Unsupported EMBEDDER_BACKEND={EMBEDDER_BACKEND}. "
                "Use 'simcse' or 'sentence-transformers'."
            )
    return _embed_model


def _embed_model_source() -> str:
    return EMBED_MODEL_PATH if os.path.exists(EMBED_MODEL_PATH) else BASE_MODEL_EMBED


def _index_config_signature() -> str:
    payload = {
        "embedding_model": _embed_model_source(),
        "embedder_backend": EMBEDDER_BACKEND,
        "embedding_pipeline": EMBEDDING_PIPELINE_VERSION,
        "text_processing": TEXT_PROCESSING_VERSION,
        "simcse_pooling": SIMCSE_POOLING,
        "chunk_size": CHUNK_SIZE,
        "chunk_overlap": CHUNK_OVERLAP,
    }
    raw = json.dumps(payload, ensure_ascii=False, sort_keys=True)
    return hashlib.sha256(raw.encode("utf-8")).hexdigest()


def get_summary_pipeline():
    global _summary_pipeline
    if _summary_pipeline is None and os.path.exists(SUMMARY_MODEL_PATH):
        from transformers import pipeline
        _summary_pipeline = pipeline(
            "summarization", model=SUMMARY_MODEL_PATH,
            tokenizer=SUMMARY_MODEL_PATH, max_new_tokens=128, device=-1,
        )
    return _summary_pipeline


def get_classifier():
    global _classifier, _label_map
    if _classifier is None and os.path.exists(CLASSIFY_MODEL_PATH):
        from transformers import pipeline
        _classifier = pipeline(
            "text-classification", model=CLASSIFY_MODEL_PATH,
            tokenizer=CLASSIFY_MODEL_PATH, device=-1,
        )
        label_map_path = os.path.join(CLASSIFY_MODEL_PATH, "label_map.json")
        if os.path.exists(label_map_path):
            with open(label_map_path) as f:
                _label_map = json.load(f)
    return _classifier, _label_map


def _metadata_matches(meta: dict | None, where: dict | None) -> bool:
    if not where:
        return True
    if not meta:
        return False
    for key, value in where.items():
        actual = meta.get(key)
        if isinstance(value, dict) and "$in" in value:
            if actual not in value["$in"]:
                return False
        elif actual != value:
            return False
    return True


class PineconeCollectionAdapter:
    """Small collection facade over Pinecone plus a local chunk cache for BM25."""

    FETCH_BATCH_SIZE = 50
    UPSERT_BATCH_SIZE = 50
    MAX_UPSERT_RETRIES = 3
    MAX_FETCH_RETRIES = 3
    DELETE_BATCH_SIZE = 1000

    def __init__(self, index, namespace: str, cache_path: str) -> None:
        self.index = index
        self.namespace = namespace
        self.cache_path = cache_path
        self._cache: dict[str, dict] | None = None

    def _load_cache(self) -> dict[str, dict]:
        if self._cache is not None:
            return self._cache
        try:
            with open(self.cache_path, encoding="utf-8") as f:
                raw = json.load(f)
        except (FileNotFoundError, json.JSONDecodeError, OSError):
            raw = {}
        self._cache = raw if isinstance(raw, dict) else {}
        return self._cache

    def _save_cache(self) -> None:
        if self._cache is None:
            return
        os.makedirs(os.path.dirname(self.cache_path), exist_ok=True)
        tmp_path = f"{self.cache_path}.tmp"
        with open(tmp_path, "w", encoding="utf-8") as f:
            json.dump(self._cache, f, ensure_ascii=False, indent=2)
        os.replace(tmp_path, self.cache_path)

    @staticmethod
    def _clean_metadata(meta: dict | None) -> dict:
        return {k: v for k, v in (meta or {}).items() if v is not None}

    def _upsert_with_retry(self, vectors: list[dict]) -> None:
        for attempt in range(1, self.MAX_UPSERT_RETRIES + 1):
            try:
                self.index.upsert(vectors=vectors, namespace=self.namespace)
                return
            except Exception as exc:
                if attempt >= self.MAX_UPSERT_RETRIES:
                    raise
                wait_seconds = 2 ** (attempt - 1)
                logger.warning(
                    "Pinecone upsert failed (%s/%s): %s; retrying in %ss",
                    attempt,
                    self.MAX_UPSERT_RETRIES,
                    exc,
                    wait_seconds,
                )
                time.sleep(wait_seconds)

    def _fetch_with_retry(self, ids: list[str]):
        for attempt in range(1, self.MAX_FETCH_RETRIES + 1):
            try:
                return self.index.fetch(ids=ids, namespace=self.namespace)
            except Exception as exc:
                if attempt >= self.MAX_FETCH_RETRIES:
                    raise
                wait_seconds = 2 ** (attempt - 1)
                logger.warning(
                    "Pinecone fetch failed (%s/%s): %s; retrying in %ss",
                    attempt,
                    self.MAX_FETCH_RETRIES,
                    exc,
                    wait_seconds,
                )
                time.sleep(wait_seconds)

    def count(self) -> int:
        try:
            stats = self.index.describe_index_stats()
            namespaces = getattr(stats, "namespaces", None)
            if namespaces is None and isinstance(stats, dict):
                namespaces = stats.get("namespaces", {})
            ns_stats = namespaces.get(self.namespace) if namespaces else None
            if ns_stats is None:
                return 0
            if isinstance(ns_stats, dict):
                return int(ns_stats.get("vector_count", 0))
            return int(getattr(ns_stats, "vector_count", 0))
        except Exception:
            return len(self._load_cache())

    def add(self, ids: list[str], embeddings: list, documents: list[str], metadatas: list[dict]) -> None:
        vectors = []
        cache = self._load_cache()
        for id_, embedding, document, metadata in zip(ids, embeddings, documents, metadatas):
            meta = self._clean_metadata(metadata)
            vectors.append({"id": id_, "values": embedding, "metadata": {**meta, "document": document}})
            cache[id_] = {"document": document, "metadata": meta}
        if vectors:
            for start in range(0, len(vectors), self.UPSERT_BATCH_SIZE):
                self._upsert_with_retry(vectors[start : start + self.UPSERT_BATCH_SIZE])
            self._save_cache()

    def update(self, ids: list[str], embeddings: list, documents: list[str], metadatas: list[dict]) -> None:
        self.add(ids=ids, embeddings=embeddings, documents=documents, metadatas=metadatas)

    def delete(self, ids: list[str]) -> None:
        if not ids:
            return
        for start in range(0, len(ids), self.DELETE_BATCH_SIZE):
            try:
                self.index.delete(ids=ids[start : start + self.DELETE_BATCH_SIZE], namespace=self.namespace)
            except Exception as exc:
                if "Namespace not found" not in str(exc):
                    raise
                logger.info("Pinecone namespace '%s' does not exist yet; skipping stale delete.", self.namespace)
                break
        cache = self._load_cache()
        changed = False
        for id_ in ids:
            changed = cache.pop(id_, None) is not None or changed
        if changed:
            self._save_cache()

    def get(
        self,
        ids: list[str] | None = None,
        include: list[str] | None = None,
        where: dict | None = None,
        limit: int | None = None,
    ) -> dict:
        include = include or []
        cache = self._load_cache()
        result_ids: list[str] = []
        documents: list[str] = []
        metadatas: list[dict] = []

        if ids is not None:
            vectors = {}
            for start in range(0, len(ids), self.FETCH_BATCH_SIZE):
                batch_ids = ids[start : start + self.FETCH_BATCH_SIZE]
                fetch_response = self._fetch_with_retry(batch_ids) if batch_ids else None
                batch_vectors = getattr(fetch_response, "vectors", None) if fetch_response is not None else {}
                if batch_vectors is None and isinstance(fetch_response, dict):
                    batch_vectors = fetch_response.get("vectors", {})
                vectors.update(batch_vectors or {})
            for id_ in ids:
                vector = vectors.get(id_) if vectors else None
                cached = cache.get(id_)
                metadata = getattr(vector, "metadata", None) if vector is not None else None
                if metadata is None and isinstance(vector, dict):
                    metadata = vector.get("metadata")
                if vector is None and cached is None:
                    continue
                document = (metadata or {}).get("document") if metadata else None
                if document is None and cached:
                    document = cached.get("document", "")
                meta = self._clean_metadata(metadata or (cached or {}).get("metadata", {}))
                meta.pop("document", None)
                if not _metadata_matches(meta, where):
                    continue
                result_ids.append(id_)
                if "documents" in include:
                    documents.append(document or "")
                if "metadatas" in include:
                    metadatas.append(meta)
                if limit and len(result_ids) >= limit:
                    break
        else:
            for id_, row in cache.items():
                meta = self._clean_metadata(row.get("metadata", {}))
                if not _metadata_matches(meta, where):
                    continue
                result_ids.append(id_)
                if "documents" in include:
                    documents.append(row.get("document", ""))
                if "metadatas" in include:
                    metadatas.append(meta)
                if limit and len(result_ids) >= limit:
                    break

        response = {"ids": result_ids}
        if "documents" in include:
            response["documents"] = documents
        if "metadatas" in include:
            response["metadatas"] = metadatas
        return response

    def query(
        self,
        query_embeddings: list,
        n_results: int,
        include: list[str] | None = None,
        where: dict | None = None,
    ) -> dict:
        response = self.index.query(
            vector=query_embeddings[0],
            top_k=n_results,
            namespace=self.namespace,
            filter=where,
            include_metadata=True,
        )
        matches = getattr(response, "matches", None)
        if matches is None and isinstance(response, dict):
            matches = response.get("matches", [])

        ids: list[str] = []
        metadatas: list[dict] = []
        distances: list[float] = []
        documents: list[str] = []
        for match in matches or []:
            match_id = getattr(match, "id", None) if not isinstance(match, dict) else match.get("id")
            score = getattr(match, "score", None) if not isinstance(match, dict) else match.get("score")
            metadata = getattr(match, "metadata", None) if not isinstance(match, dict) else match.get("metadata")
            meta = self._clean_metadata(metadata)
            document = meta.pop("document", "")
            ids.append(match_id)
            metadatas.append(meta)
            distances.append(1 - float(score or 0))
            documents.append(document)

        result = {"ids": [ids]}
        include = include or []
        if "metadatas" in include:
            result["metadatas"] = [metadatas]
        if "distances" in include:
            result["distances"] = [distances]
        if "documents" in include:
            result["documents"] = [documents]
        return result


def _get_pinecone_collection() -> PineconeCollectionAdapter:
    if not PINECONE_API_KEY:
        raise RuntimeError("PINECONE_API_KEY is required when VECTOR_DB=pinecone.")
    from pinecone import Pinecone, ServerlessSpec

    pc = Pinecone(api_key=PINECONE_API_KEY)
    if not pc.has_index(PINECONE_INDEX_NAME):
        pc.create_index(
            name=PINECONE_INDEX_NAME,
            dimension=EMBEDDING_DIM,
            metric="cosine",
            spec=ServerlessSpec(cloud=PINECONE_CLOUD, region=PINECONE_REGION),
            deletion_protection="disabled",
        )
    for _ in range(60):
        desc = pc.describe_index(PINECONE_INDEX_NAME)
        status = getattr(desc, "status", None)
        if status is None and isinstance(desc, dict):
            status = desc.get("status", {})
        ready = status.get("ready") if isinstance(status, dict) else getattr(status, "ready", False)
        if ready:
            dimension = desc.get("dimension") if isinstance(desc, dict) else getattr(desc, "dimension", None)
            if dimension and int(dimension) != EMBEDDING_DIM:
                raise RuntimeError(
                    f"Pinecone index dimension mismatch: {dimension} != EMBEDDING_DIM={EMBEDDING_DIM}."
                )
            break
        time.sleep(1)
    else:
        raise RuntimeError(f"Pinecone index is not ready: {PINECONE_INDEX_NAME}")
    return PineconeCollectionAdapter(
        index=pc.Index(PINECONE_INDEX_NAME),
        namespace=PINECONE_NAMESPACE,
        cache_path=PINECONE_CACHE_PATH,
    )


def get_vector_collection():
    global _vector_collection
    if _vector_collection is None:
        if VECTOR_DB != "pinecone":
            raise RuntimeError(
                f"Pinecone is required for the production vector store; got VECTOR_DB={VECTOR_DB!r}."
            )
        _vector_collection = _get_pinecone_collection()
    return _vector_collection


def classify_notice(title: str, body: str) -> str:
    clf, label_map = get_classifier()
    if clf is None:
        return infer_category(title, body)
    try:
        result   = clf(f"{title} {body[:200]}", truncation=True)[0]
        label_id = result["label"].replace("LABEL_", "")
        return label_map.get(label_id, "기타")
    except Exception:
        return infer_category(title, body)


def load_notices_cache() -> list[dict]:
    if os.path.exists(NOTICES_CACHE_PATH):
        with open(NOTICES_CACHE_PATH, encoding="utf-8") as f:
            return json.load(f)
    return []


def _load_index_manifest() -> dict:
    if not os.path.exists(INDEX_MANIFEST_PATH):
        return {"version": 1, "revision": 0, "index_config": {}, "notices": {}}
    try:
        with open(INDEX_MANIFEST_PATH, encoding="utf-8") as f:
            manifest = json.load(f)
    except (json.JSONDecodeError, OSError):
        logger.warning("인덱스 manifest를 읽지 못해 전체 갱신 대상으로 처리합니다.")
        return {"version": 1, "revision": 0, "index_config": {}, "notices": {}}
    manifest.setdefault("version", 1)
    manifest.setdefault("revision", 0)
    manifest.setdefault("index_config", {})
    manifest.setdefault("notices", {})
    return manifest


def _save_index_manifest(manifest: dict) -> None:
    os.makedirs(os.path.dirname(INDEX_MANIFEST_PATH), exist_ok=True)
    manifest["revision"] = int(manifest.get("revision", 0)) + 1
    tmp_path = f"{INDEX_MANIFEST_PATH}.tmp"
    with open(tmp_path, "w", encoding="utf-8") as f:
        json.dump(manifest, f, ensure_ascii=False, indent=2)
    os.replace(tmp_path, INDEX_MANIFEST_PATH)


def get_index_fingerprint() -> tuple[int, int, str]:
    """Return a cheap fingerprint for search-side cache invalidation."""
    manifest = _load_index_manifest()
    return (
        get_vector_collection().count(),
        int(manifest.get("revision", 0)),
        manifest.get("index_config", {}).get("signature", ""),
    )


def _notice_doc_id(url: str) -> str:
    return hashlib.md5(url.encode()).hexdigest()


def _notice_content_hash(item: dict) -> str:
    payload = {
        "url": item.get("url", ""),
        "title": item.get("title", ""),
        "body": item.get("body", ""),
        "date": item.get("date", ""),
        "category": item.get("category", ""),
    }
    raw = json.dumps(payload, ensure_ascii=False, sort_keys=True)
    return hashlib.sha256(raw.encode("utf-8")).hexdigest()


def index_notices(
    notices: list[dict],
    force: bool = False,
    sync_deletions: bool = False,
    notice_batch_size: int = 20,
    embed_batch_size: int = 16,
) -> int:
    """Index notices into the configured vector DB. Returns count of newly indexed notices."""
    collection = get_vector_collection()
    manifest   = _load_index_manifest()
    manifest_notices = manifest["notices"]
    notice_batch_size = max(1, notice_batch_size)
    embed_batch_size = max(1, embed_batch_size)
    index_config = {
        "signature": _index_config_signature(),
        "embedding_model": _embed_model_source(),
        "embedder_backend": EMBEDDER_BACKEND,
        "embedding_pipeline": EMBEDDING_PIPELINE_VERSION,
        "text_processing": TEXT_PROCESSING_VERSION,
        "simcse_pooling": SIMCSE_POOLING,
        "chunk_size": CHUNK_SIZE,
        "chunk_overlap": CHUNK_OVERLAP,
    }
    config_changed = manifest.get("index_config", {}).get("signature") != index_config["signature"]
    if config_changed:
        logger.info("인덱스 설정이 변경되어 전체 재색인 대상으로 처리합니다.")
        manifest["index_config"] = index_config

    # 1. 전체 doc_id/content_hash 계산
    all_records = [
        (_notice_doc_id(item["url"]), _notice_content_hash(item), item)
        for item in notices
    ]
    current_doc_ids = {doc_id for doc_id, _, _ in all_records}
    current_urls = {item["url"] for _, _, item in all_records}
    existing_ids_by_url: dict[str, list[str]] = {}

    stale_chunk_ids: list[str] = []
    if sync_deletions:
        existing = collection.get(include=["metadatas"])
        for existing_id, meta in zip(existing["ids"], existing["metadatas"]):
            if meta and meta.get("url"):
                existing_ids_by_url.setdefault(meta["url"], []).append(existing_id)
        for url, ids in existing_ids_by_url.items():
            if url not in current_urls:
                stale_chunk_ids.extend(ids)

        stale_doc_ids = set(manifest_notices) - current_doc_ids
        for stale_doc_id in stale_doc_ids:
            entry = manifest_notices.get(stale_doc_id, {})
            stale_chunk_ids.extend(entry.get("chunk_ids", []))
            stale_chunk_ids.append(stale_doc_id)
        if stale_chunk_ids:
            existing_stale = set(collection.get(ids=list(set(stale_chunk_ids)))["ids"])
            if existing_stale:
                collection.delete(ids=list(existing_stale))
        for stale_doc_id in stale_doc_ids:
            manifest_notices.pop(stale_doc_id, None)
        if stale_doc_ids:
            logger.info("소스에서 사라진 공지 %d건을 인덱스 manifest에서 제거했습니다.", len(stale_doc_ids))

    # 2. 존재 확인: N번 개별 쿼리 → 1번 배치 쿼리
    chunk0_ids      = [f"{doc_id}_0" for doc_id, _, _ in all_records]
    existing_chunk0 = set(collection.get(ids=chunk0_ids)["ids"]) if chunk0_ids else set()

    pending: list[tuple[str, str, dict]] = []
    for doc_id, content_hash, item in all_records:
        indexed = f"{doc_id}_0" in existing_chunk0
        manifest_entry = manifest_notices.get(doc_id)
        unchanged = (
            manifest_entry
            and manifest_entry.get("content_hash") == content_hash
            and manifest_entry.get("index_config_signature") == index_config["signature"]
            and indexed
        )
        if force or config_changed or not unchanged:
            pending.append((doc_id, content_hash, item))

    # 3. 구버전 포맷(청크 없는 doc_id) 일괄 삭제 — 1번 쿼리
    if pending:
        old_ids      = [doc_id for doc_id, _, _ in pending]
        existing_old = set(collection.get(ids=old_ids)["ids"])
        to_delete    = [did for did in old_ids if did in existing_old]
        if to_delete:
            collection.delete(ids=to_delete)

    if not pending:
        if sync_deletions and stale_chunk_ids:
            _save_index_manifest(manifest)
        return 0

    if not existing_ids_by_url and any(
        not manifest_notices.get(doc_id, {}).get("chunk_ids")
        for doc_id, _, _ in pending
    ):
        existing = collection.get(include=["metadatas"])
        for existing_id, meta in zip(existing["ids"], existing["metadatas"]):
            if meta and meta.get("url"):
                existing_ids_by_url.setdefault(meta["url"], []).append(existing_id)

    # pending이 있을 때만 무거운 임베딩 모델을 로드한다.
    model = get_embed_model()
    logger.info(
        "임베딩 시작! 총 %d개 공지 처리 예정 (notice_batch=%d, embed_batch=%d)",
        len(pending),
        notice_batch_size,
        embed_batch_size,
    )

    total_indexed = 0

    for batch_start in range(0, len(pending), notice_batch_size):
        batch = pending[batch_start : batch_start + notice_batch_size]
        batch_no = batch_start // notice_batch_size + 1
        total_batches = (len(pending) + notice_batch_size - 1) // notice_batch_size

        notice_chunks: list[tuple[str, str, list[str], list[str], list[dict]]] = []
        batch_delete_ids: list[str] = []
        docs: list[str] = []

        for doc_id, content_hash, item in batch:
            entry = manifest_notices.get(doc_id, {})
            ids_to_delete = entry.get("chunk_ids", [])
            if not ids_to_delete:
                ids_to_delete = existing_ids_by_url.get(item["url"], [])
            if ids_to_delete:
                batch_delete_ids.extend(ids_to_delete)

            body     = item.get("body", "")
            inferred_category = classify_notice(item["title"], body)
            existing_category = item.get("category")
            category = (
                inferred_category
                if inferred_category == "봉사/서포터즈" and existing_category in {None, "", "국제교류", "기타"}
                else existing_category or inferred_category
            )
            chunks   = chunk_text(f"제목: {item['title']}\n\n{body}")
            meta     = {"title": item["title"], "url": item["url"],
                        "date": item["date"], "category": category}
            chunk_ids = [f"{doc_id}_{i}" for i in range(len(chunks))]
            notice_chunks.append(
                (doc_id, content_hash, chunk_ids, chunks, [meta] * len(chunks))
            )
            docs.extend(chunks)

        if batch_delete_ids:
            collection.delete(ids=list(dict.fromkeys(batch_delete_ids)))

        logger.info(
            "공지 배치 인코딩 시작: %d/%d 배치 (%d개 공지, %d개 청크, device=%s, embed_batch=%d)",
            batch_no,
            total_batches,
            len(batch),
            len(docs),
            getattr(model, "_device", getattr(model, "device", "unknown")),
            embed_batch_size,
        )
        embeddings = model.encode(
            docs,
            batch_size=embed_batch_size,
            show_progress_bar=True,
        ).tolist()

        ids: list[str] = []
        metas: list[dict] = []
        for _, _, n_ids, _, n_metas in notice_chunks:
            ids.extend(n_ids)
            metas.extend(n_metas)

        collection.add(
            ids=ids,
            embeddings=embeddings,
            documents=docs,
            metadatas=metas,
        )
        total_indexed += len(batch)
        for doc_id, content_hash, n_ids, _, _ in notice_chunks:
            manifest_notices[doc_id] = {
                "content_hash": content_hash,
                "index_config_signature": index_config["signature"],
                "chunk_ids": n_ids,
            }
        _save_index_manifest(manifest)
        del embeddings, ids, metas, docs, notice_chunks
        _release_torch_cache()
        logger.info(
            "공지 배치 저장 완료: %d/%d 배치, 누적 %d/%d개 공지",
            batch_no,
            total_batches,
            total_indexed,
            len(pending),
        )

    return total_indexed