spark_storage/cache_peer/
mod.rs

1// SPDX-License-Identifier: AGPL-3.0-only
2//
3// KV overflow blade — a dumb remote-RAM tier for the high-speed-swap KV cache.
4//
5// Where `expert_peer` serves READ-ONLY expert weights over one-sided RDMA READ,
6// this serves a READ-WRITE slab of RAM: a streaming client OFFLOADS cold K/V
7// groups into it with `IBV_WR_RDMA_WRITE` and RESTORES them with
8// `IBV_WR_RDMA_READ`, both one-sided, peer CPU idle. It is the "faster than the
9// SSD" overflow tier: local pinned RAM → **peer RAM (~12 GB/s over CX7)** →
10// local NVMe/USB SSD (~2 GB/s). The peer owns nothing — each group belongs to
11// exactly one client sequence; this process is a passive memory blade.
12//
13// Addressing is the flat group-id space of `GroupLayout`: a group lands at
14// `base + group_id * group_stride`, so no per-group bookkeeping on the peer.
15//
16// Wire protocol (little-endian), connection-oriented:
17//   1. client -> [u64 total_bytes]  (num_groups * group_stride it will address)
18//   2. peer allocates + registers a RW MR of that size, replies with
19//      CacheServerParams [u32 qpn][u32 psn][16 gid][u64 base_addr][u32 rkey]
20//   3. client -> VerbsClientParams [u32 qpn][u32 psn][16 gid]
21//   4. peer connects its QP, replies [u8 STATUS_OK]
22//   5. client does one-sided WRITE/READ; peer idles until the client hangs up,
23//      then unregisters + unmaps the blade.
24//
25// This module is split per the Atlas SDD file-size idiom:
26//   `server_impl.rs` — accept loop, first-u64 dispatch, server-side rail
27//                      handshake holding the crate's SINGLE `reg_mr_rw` call
28//                      site (the access flag stays AT the call site — census-
29//                      pinned by `tests/reg_mr_flag_audit.rs`), both data
30//                      planes;
31//   `registry.rs`    — peer POLICY: the process-global (kind, blob_bytes)
32//                      paging-arena registry, the disk-cap carve, and the anon
33//                      `Mmap` RAII the RDMA-registered arenas live in.
34// The peer's residency IS `atlas_tier::Residency` and its verbs ARE
35// `atlas_rdma::verbs`, imported directly — nothing tier- or verbs-shaped is
36// hand-rolled here.
37
38// The RW-blade handshake codec lives verbatim in the CUDA-free `atlas-rdma`
39// crate; re-exported at its old path so the server below and both RW clients
40// are zero-diff. Byte layout golden-pinned in `tests/rdma_wire_golden.rs` —
41// it is what the fleet cache-peer binary speaks.
42pub use atlas_rdma::wire::CacheServerParams;
43
44// Peer paging policy — verbs-only, like the paging handshake it backs.
45#[cfg(atlas_rdma_verbs)]
46mod registry;
47#[cfg(unix)]
48mod server_impl;
49
50#[cfg(unix)]
51pub use server_impl::{RdmaConfig, serve};
52
53// `kv_server_params_round_trip` lives WITH the codec in
54// `crates/atlas-rdma/tests/wire_roundtrip.rs`; the exact byte layout stays
55// pinned by `tests/rdma_wire_golden.rs` here. The `carve_disk_slots`
56// precedence pins live in `registry_tests.rs`.
spark_storage/cache_peer/mod.rs

spark_storage/cache_peer/
mod.rs
