File size: 1,757 Bytes
119e586 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | (ns snapkitty.lisp.knowledge.store
"Document store with rate limiting"
(:require [promesa.core :as p]
[snapkitty.lisp.knowledge.qdrant :as qdrant]
[snapkitty.lisp.knowledge.embedding :as embedding]
[snapkitty.lisp.mcp.util :as util]))
;; Rate limiter: 10 docs/sec (token bucket)
(defonce rate-limiter (atom {:capacity 10 :tokens 10 :last-refill (js/Date.now)}))
(defn check-rate-limit! []
"Check and update rate limit — token bucket"
(let [now (js/Date.now)
{:keys [tokens capacity last-refill]} @rate-limiter
elapsed (/ (- now last-refill) 1000)
refilled (min capacity (+ tokens (* elapsed 10)))]
(if (< refilled 1)
(throw (ex-info "Rate limit exceeded" {:retry-after 0.1}))
(do
(swap! rate-limiter assoc
:tokens (dec refilled)
:last-refill now)
true))))
(defn upsert! [id doc-data]
"Store document with embedding — RATE LIMITED"
(check-rate-limit!)
(p/let [vector (embedding/embed (:content doc-data))
_ (qdrant/upsert-point! {}
"snapkitty-knowledge"
id
{:vector vector
:payload doc-data})]
{:id id :stored true}))
(defn search [query {:keys [limit tags] :or {limit 10}}]
"Semantic search — tags optional filter"
(p/let [query-vector (embedding/embed query)
results (qdrant/search-points {} "snapkitty-knowledge" query-vector
{:limit limit})]
results))
(defn delete! [id]
"Delete document by ID"
(qdrant/delete-point! {} "snapkitty-knowledge" id))
|