{
  "annotation_id": "cb9cc8ce-2500-449d-b02b-7330e4a33ab4",
  "date_created": "2026-02-17T05:53:05.066000Z",
  "date_modified": "2026-02-17T05:53:05.066000Z",
  "file_hash": "84fd29a1724f6a65ac2f838b56c65733647770a8a6a38ff08539cbab8a9d0715",
  "private": false,
  "record": {
    "abstract": "Modern Retrieval-Augmented Generation (RAG) systems struggle with a fundamental architectural tension: vector indices are optimized for query latency but poorly handle continuous knowledge updates, while data lakes excel at versioning but introduce query latency penalties. We introduce LiveVectorLake, a dual-tier temporal knowledge base architecture that enables real-time semantic search on current knowledge while maintaining complete version history for compliance, auditability, and point-in-time retrieval. The system introduces three core architectural contributions: (1) Content-addressable chunk-level synchronization using SHA-256 hashing for deterministic change detection without external state tracking; (2) Dual-tier storage separating hot-tier vector indices (Milvus with HNSW) from cold-tier columnar versioning (Delta Lake with Parquet), optimizing query latency and storage cost independently; (3) Temporal query routing enabling point-in-time knowledge retrieval via delta-versioning with ACID consistency across tiers. Evaluation on a 100-document corpus versioned across five time points demonstrates: (i) 10-15% re-processing of content during updates compared to 100% for full re-indexing; (ii) sub-100ms retrieval latency on current knowledge; (iii) sub-2s latency for temporal queries across version history; and (iv) storage cost optimization through hot/cold tier separation (only current chunks in expensive vector indices). The approach enables production RAG deployments requiring simultaneous optimization for query performance, update efficiency, and regulatory compliance. Code and resources: [https://github.com/praj-tarun/LiveVectorLake]",
    "arxiv_id": "2601.05270",
    "authors": [
      "Tarun Prajapati"
    ],
    "categories": [
      "cs.IR",
      "cs.AI",
      "cs.DB"
    ],
    "license": "http://creativecommons.org/licenses/by/4.0/",
    "title": "LiveVectorLake: A Real-Time Versioned Knowledge Base Architecture for Streaming Vector Updates and Temporal Retrieval",
    "url": "https://arxiv.org/abs/2601.05270",
    "version": "v1"
  },
  "schema_id": "dorsal/arxiv",
  "source": {
    "execution_id": "f230732f-3a91-4b01-ba06-ab0457c0d7f5",
    "id": "arXiv Dataset",
    "type": "Model",
    "variant": "snapshot-2026-01-17",
    "version": "0.1.0"
  },
  "user_id": 1000002
}