test(store): add compaction, scan, snapshot, and durability tests
This commit is contained in:
dualfroz committed
1 parent
ffc4fadc47
commit
c593d2459d
1 file changed
+324
+324
@@ -1148,3 +1148,327 @@ test "write batch applies all or nothing" {
|
||||
try std.testing.expectEqual(@as(?[]u8, null), try db.get(allocator, "existing"));
|
||||
}
|
||||
|
||||
test "block cache serves repeated reads" {
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
var db = try openTemp(&tmp, allocator, .{});
|
||||
defer db.close();
|
||||
|
||||
try db.put("alpha", "one");
|
||||
try db.put("bravo", "two");
|
||||
try db.flush();
|
||||
|
||||
const first = (try db.get(allocator, "alpha")).?;
|
||||
allocator.free(first);
|
||||
try std.testing.expect(db.blockCache.stats.misses >= 1);
|
||||
|
||||
const hitsBefore = db.blockCache.stats.hits;
|
||||
const second = (try db.get(allocator, "alpha")).?;
|
||||
allocator.free(second);
|
||||
try std.testing.expect(db.blockCache.stats.hits > hitsBefore);
|
||||
}
|
||||
|
||||
test "leveled compaction keeps levels disjoint and drops bottom tombstones" {
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
var db = try openTemp(&tmp, allocator, .{
|
||||
.flushThreshold = 512,
|
||||
.l0CompactionTrigger = 2,
|
||||
.l1BaseBytes = 4 * 1024 * 1024,
|
||||
.sstableTargetBytes = 2048,
|
||||
});
|
||||
defer db.close();
|
||||
|
||||
var keyBuf: [16]u8 = undefined;
|
||||
const payload = "x" ** 120;
|
||||
var i: usize = 0;
|
||||
while (i < 60) : (i += 1) {
|
||||
const key = try std.fmt.bufPrint(&keyBuf, "key:{d:0>4}", .{i});
|
||||
try db.put(key, payload);
|
||||
}
|
||||
try db.delete("key:0005");
|
||||
try db.delete("key:0025");
|
||||
try db.flush();
|
||||
|
||||
try db.maybeCompact();
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 0), db.countLevel(0));
|
||||
try std.testing.expect(db.sstables.items.len >= 2);
|
||||
|
||||
var live: usize = 0;
|
||||
var totalEntries: u64 = 0;
|
||||
for (db.sstables.items) |reader| {
|
||||
try std.testing.expect(reader.level >= 1);
|
||||
totalEntries += reader.numEntries;
|
||||
}
|
||||
|
||||
var levelOneReaders = std.ArrayList(*Reader).init(allocator);
|
||||
defer levelOneReaders.deinit();
|
||||
for (db.sstables.items) |reader| {
|
||||
if (reader.level == 1) try levelOneReaders.append(reader);
|
||||
}
|
||||
std.mem.sort(*Reader, levelOneReaders.items, {}, lessByMinKey);
|
||||
var r: usize = 1;
|
||||
while (r < levelOneReaders.items.len) : (r += 1) {
|
||||
try std.testing.expect(order(levelOneReaders.items[r - 1].maxKey, levelOneReaders.items[r].minKey) == .lt);
|
||||
}
|
||||
|
||||
i = 0;
|
||||
while (i < 60) : (i += 1) {
|
||||
const key = try std.fmt.bufPrint(&keyBuf, "key:{d:0>4}", .{i});
|
||||
if (i == 5 or i == 25) {
|
||||
try std.testing.expectEqual(@as(?[]u8, null), try db.get(allocator, key));
|
||||
} else {
|
||||
const got = (try db.get(allocator, key)).?;
|
||||
defer allocator.free(got);
|
||||
try std.testing.expectEqualStrings(payload, got);
|
||||
live += 1;
|
||||
}
|
||||
}
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 58), live);
|
||||
try std.testing.expectEqual(@as(u64, 58), totalEntries);
|
||||
}
|
||||
|
||||
fn lessByMinKey(_: void, a: *Reader, b: *Reader) bool {
|
||||
return order(a.minKey, b.minKey) == .lt;
|
||||
}
|
||||
|
||||
fn lessString(_: void, a: []const u8, b: []const u8) bool {
|
||||
return std.mem.lessThan(u8, a, b);
|
||||
}
|
||||
|
||||
test "reverse scan yields descending order merging sources" {
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
var db = try openTemp(&tmp, allocator, .{ .flushThreshold = 48 });
|
||||
defer db.close();
|
||||
|
||||
var keyBuf: [8]u8 = undefined;
|
||||
var i: usize = 0;
|
||||
while (i < 26) : (i += 1) {
|
||||
const key = try std.fmt.bufPrint(&keyBuf, "{c}", .{@as(u8, 'a') + @as(u8, @intCast(i))});
|
||||
try db.put(key, "old");
|
||||
}
|
||||
try db.put("m", "fresh");
|
||||
try db.delete("d");
|
||||
|
||||
var iter = try db.scanReverse(allocator, .{});
|
||||
defer iter.deinit();
|
||||
var prev: [8]u8 = undefined;
|
||||
var prevLen: usize = 0;
|
||||
var count: usize = 0;
|
||||
var sawFreshM = false;
|
||||
var sawD = false;
|
||||
while (try iter.next()) |entry| {
|
||||
if (prevLen > 0) try std.testing.expect(order(entry.key, prev[0..prevLen]) == .lt);
|
||||
@memcpy(prev[0..entry.key.len], entry.key);
|
||||
prevLen = entry.key.len;
|
||||
if (std.mem.eql(u8, entry.key, "m")) {
|
||||
try std.testing.expectEqualStrings("fresh", entry.value);
|
||||
sawFreshM = true;
|
||||
}
|
||||
if (std.mem.eql(u8, entry.key, "d")) sawD = true;
|
||||
count += 1;
|
||||
}
|
||||
try std.testing.expectEqual(@as(usize, 25), count);
|
||||
try std.testing.expect(sawFreshM);
|
||||
try std.testing.expect(!sawD);
|
||||
}
|
||||
|
||||
test "prefix scan isolates a namespace" {
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
var db = try openTemp(&tmp, allocator, .{ .flushThreshold = 64 });
|
||||
defer db.close();
|
||||
|
||||
try db.put("user:alice", "1");
|
||||
try db.put("user:bob", "2");
|
||||
try db.put("user:carol", "3");
|
||||
try db.put("post:1", "x");
|
||||
try db.put("userx", "y");
|
||||
try db.put("zzz", "z");
|
||||
|
||||
var iter = try db.scanPrefix(allocator, "user:");
|
||||
defer iter.deinit();
|
||||
const expected = [_][]const u8{ "user:alice", "user:bob", "user:carol" };
|
||||
var idx: usize = 0;
|
||||
while (try iter.next()) |entry| {
|
||||
try std.testing.expect(idx < expected.len);
|
||||
try std.testing.expectEqualStrings(expected[idx], entry.key);
|
||||
idx += 1;
|
||||
}
|
||||
try std.testing.expectEqual(expected.len, idx);
|
||||
}
|
||||
|
||||
test "inclusive and exclusive bounds" {
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
var db = try openTemp(&tmp, allocator, .{});
|
||||
defer db.close();
|
||||
|
||||
var keyBuf: [8]u8 = undefined;
|
||||
var i: usize = 0;
|
||||
while (i < 10) : (i += 1) {
|
||||
const key = try std.fmt.bufPrint(&keyBuf, "k{d}", .{i});
|
||||
try db.put(key, "v");
|
||||
}
|
||||
|
||||
{
|
||||
var iter = try db.scanRange(allocator, .{ .start = "k2", .end = "k5", .startInclusive = true, .endInclusive = true });
|
||||
defer iter.deinit();
|
||||
const expected = [_][]const u8{ "k2", "k3", "k4", "k5" };
|
||||
var idx: usize = 0;
|
||||
while (try iter.next()) |entry| {
|
||||
try std.testing.expectEqualStrings(expected[idx], entry.key);
|
||||
idx += 1;
|
||||
}
|
||||
try std.testing.expectEqual(expected.len, idx);
|
||||
}
|
||||
{
|
||||
var iter = try db.scanRange(allocator, .{ .start = "k2", .end = "k5", .startInclusive = false, .endInclusive = false });
|
||||
defer iter.deinit();
|
||||
const expected = [_][]const u8{ "k3", "k4" };
|
||||
var idx: usize = 0;
|
||||
while (try iter.next()) |entry| {
|
||||
try std.testing.expectEqualStrings(expected[idx], entry.key);
|
||||
idx += 1;
|
||||
}
|
||||
try std.testing.expectEqual(expected.len, idx);
|
||||
}
|
||||
}
|
||||
|
||||
fn setLive(map: *std.StringHashMap([]const u8), arena: Allocator, key: []const u8, value: []const u8) !void {
|
||||
const gop = try map.getOrPut(key);
|
||||
if (!gop.found_existing) gop.key_ptr.* = try arena.dupe(u8, key);
|
||||
gop.value_ptr.* = try arena.dupe(u8, value);
|
||||
}
|
||||
|
||||
test "scan matches sorted reference across flush restart and compaction" {
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
var refArena = std.heap.ArenaAllocator.init(allocator);
|
||||
defer refArena.deinit();
|
||||
const arena = refArena.allocator();
|
||||
|
||||
var live = std.StringHashMap([]const u8).init(allocator);
|
||||
defer live.deinit();
|
||||
|
||||
const keyspace = 300;
|
||||
const ops = 4000;
|
||||
var prng = std.Random.DefaultPrng.init(0xC0BA17);
|
||||
const random = prng.random();
|
||||
|
||||
{
|
||||
var db = try openTemp(&tmp, allocator, .{ .flushThreshold = 2048, .sstableTargetBytes = 1024 });
|
||||
defer db.close();
|
||||
|
||||
var keyBuf: [16]u8 = undefined;
|
||||
var valBuf: [24]u8 = undefined;
|
||||
var op: usize = 0;
|
||||
while (op < ops) : (op += 1) {
|
||||
const id = random.intRangeLessThan(usize, 0, keyspace);
|
||||
const key = try std.fmt.bufPrint(&keyBuf, "key:{d:0>6}", .{id});
|
||||
if (random.intRangeLessThan(usize, 0, 10) < 3) {
|
||||
try db.delete(key);
|
||||
_ = live.remove(key);
|
||||
} else {
|
||||
const value = try std.fmt.bufPrint(&valBuf, "val-{d}-{d}", .{ id, random.int(u16) });
|
||||
try db.put(key, value);
|
||||
try setLive(&live, arena, key, value);
|
||||
}
|
||||
if (random.intRangeLessThan(usize, 0, 200) == 0) try db.flush();
|
||||
}
|
||||
try db.maybeCompact();
|
||||
}
|
||||
|
||||
var db = try openTemp(&tmp, allocator, .{ .flushThreshold = 2048, .sstableTargetBytes = 1024 });
|
||||
defer db.close();
|
||||
|
||||
var sortedKeys = std.ArrayList([]const u8).init(allocator);
|
||||
defer sortedKeys.deinit();
|
||||
var keyIt = live.keyIterator();
|
||||
while (keyIt.next()) |key| try sortedKeys.append(key.*);
|
||||
std.mem.sort([]const u8, sortedKeys.items, {}, lessString);
|
||||
|
||||
var forward = try db.scan(allocator, null, null);
|
||||
defer forward.deinit();
|
||||
var idx: usize = 0;
|
||||
while (try forward.next()) |entry| {
|
||||
try std.testing.expect(idx < sortedKeys.items.len);
|
||||
try std.testing.expectEqualStrings(sortedKeys.items[idx], entry.key);
|
||||
try std.testing.expectEqualStrings(live.get(entry.key).?, entry.value);
|
||||
idx += 1;
|
||||
}
|
||||
try std.testing.expectEqual(sortedKeys.items.len, idx);
|
||||
|
||||
var reverse = try db.scanReverse(allocator, .{});
|
||||
defer reverse.deinit();
|
||||
idx = sortedKeys.items.len;
|
||||
while (try reverse.next()) |entry| {
|
||||
try std.testing.expect(idx > 0);
|
||||
idx -= 1;
|
||||
try std.testing.expectEqualStrings(sortedKeys.items[idx], entry.key);
|
||||
try std.testing.expectEqualStrings(live.get(entry.key).?, entry.value);
|
||||
}
|
||||
try std.testing.expectEqual(@as(usize, 0), idx);
|
||||
|
||||
for (sortedKeys.items) |key| {
|
||||
const got = (try db.get(allocator, key)).?;
|
||||
defer allocator.free(got);
|
||||
try std.testing.expectEqualStrings(live.get(key).?, got);
|
||||
}
|
||||
}
|
||||
|
||||
test "crash recovery honors wal durability boundary" {
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
const total = 8;
|
||||
{
|
||||
var db = try openTemp(&tmp, allocator, .{});
|
||||
defer db.close();
|
||||
var keyBuf: [8]u8 = undefined;
|
||||
var i: usize = 0;
|
||||
while (i < total) : (i += 1) {
|
||||
const key = try std.fmt.bufPrint(&keyBuf, "wkey{d}", .{i});
|
||||
try db.put(key, "durable");
|
||||
}
|
||||
try std.testing.expectEqual(@as(usize, 0), db.sstables.items.len);
|
||||
}
|
||||
|
||||
{
|
||||
const walFile = try tmp.dir.openFile("store/wal.log", .{ .mode = .read_write });
|
||||
defer walFile.close();
|
||||
const size = try walFile.getEndPos();
|
||||
try walFile.setEndPos(size - 1);
|
||||
try walFile.sync();
|
||||
}
|
||||
|
||||
var db = try openTemp(&tmp, allocator, .{});
|
||||
defer db.close();
|
||||
|
||||
var keyBuf: [8]u8 = undefined;
|
||||
var i: usize = 0;
|
||||
while (i < total - 1) : (i += 1) {
|
||||
const key = try std.fmt.bufPrint(&keyBuf, "wkey{d}", .{i});
|
||||
const got = (try db.get(allocator, key)).?;
|
||||
defer allocator.free(got);
|
||||
try std.testing.expectEqualStrings("durable", got);
|
||||
}
|
||||
const lastKey = try std.fmt.bufPrint(&keyBuf, "wkey{d}", .{total - 1});
|
||||
try std.testing.expectEqual(@as(?[]u8, null), try db.get(allocator, lastKey));
|
||||
}
|
||||
Reference in new issue
Block a user