feat(core): implement varint and byte-buffer codec utilities
This commit is contained in:
dualfroz committed
1 parent
228905a6ae
commit
e1d76bdc42
1 file changed
+469
@@ -0,0 +1,469 @@
|
||||
#include "vf/Codec.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
namespace vf {
|
||||
|
||||
void ByteWriter::putU16(uint16_t v) {
|
||||
buffer.push_back(static_cast<uint8_t>((v >> 8) & 0xFFu));
|
||||
buffer.push_back(static_cast<uint8_t>(v & 0xFFu));
|
||||
}
|
||||
|
||||
void ByteWriter::putU32(uint32_t v) {
|
||||
for (int shift = 24; shift >= 0; shift -= 8)
|
||||
buffer.push_back(static_cast<uint8_t>((v >> shift) & 0xFFu));
|
||||
}
|
||||
|
||||
void ByteWriter::putU64(uint64_t v) {
|
||||
for (int shift = 56; shift >= 0; shift -= 8)
|
||||
buffer.push_back(static_cast<uint8_t>((v >> shift) & 0xFFu));
|
||||
}
|
||||
|
||||
void ByteWriter::putVarU32(uint32_t v) {
|
||||
while (v >= 0x80u) {
|
||||
buffer.push_back(static_cast<uint8_t>(v) | 0x80u);
|
||||
v >>= 7;
|
||||
}
|
||||
buffer.push_back(static_cast<uint8_t>(v));
|
||||
}
|
||||
|
||||
void ByteWriter::putVarU64(uint64_t v) {
|
||||
while (v >= 0x80u) {
|
||||
buffer.push_back(static_cast<uint8_t>(v) | 0x80u);
|
||||
v >>= 7;
|
||||
}
|
||||
buffer.push_back(static_cast<uint8_t>(v));
|
||||
}
|
||||
|
||||
void ByteWriter::putVarI32(int32_t v) { putVarU32(zigzagEncode32(v)); }
|
||||
void ByteWriter::putVarI64(int64_t v) { putVarU64(zigzagEncode64(v)); }
|
||||
|
||||
void ByteWriter::putFloat(float v) {
|
||||
uint32_t bits;
|
||||
std::memcpy(&bits, &v, sizeof(bits));
|
||||
putU32(bits);
|
||||
}
|
||||
|
||||
void ByteWriter::putDouble(double v) {
|
||||
uint64_t bits;
|
||||
std::memcpy(&bits, &v, sizeof(bits));
|
||||
putU64(bits);
|
||||
}
|
||||
|
||||
void ByteWriter::putBytes(const uint8_t* p, std::size_t n) {
|
||||
buffer.insert(buffer.end(), p, p + n);
|
||||
}
|
||||
|
||||
void ByteWriter::putString(const std::string& s) {
|
||||
putVarU32(static_cast<uint32_t>(s.size()));
|
||||
buffer.insert(buffer.end(), s.begin(), s.end());
|
||||
}
|
||||
|
||||
bool ByteReader::need(std::size_t n) {
|
||||
if (failed || cursor + n > size) {
|
||||
failed = true;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getU8(uint8_t& out) {
|
||||
if (!need(1)) return false;
|
||||
out = data[cursor++];
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getU16(uint16_t& out) {
|
||||
if (!need(2)) return false;
|
||||
out = static_cast<uint16_t>((data[cursor] << 8) | data[cursor + 1]);
|
||||
cursor += 2;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getU32(uint32_t& out) {
|
||||
if (!need(4)) return false;
|
||||
out = 0;
|
||||
for (int i = 0; i < 4; ++i) out = (out << 8) | data[cursor + static_cast<std::size_t>(i)];
|
||||
cursor += 4;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getU64(uint64_t& out) {
|
||||
if (!need(8)) return false;
|
||||
out = 0;
|
||||
for (int i = 0; i < 8; ++i) out = (out << 8) | data[cursor + static_cast<std::size_t>(i)];
|
||||
cursor += 8;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getVarU32(uint32_t& out) {
|
||||
out = 0;
|
||||
for (int shift = 0; shift < 35; shift += 7) {
|
||||
uint8_t byte;
|
||||
if (!getU8(byte)) return false;
|
||||
out |= static_cast<uint32_t>(byte & 0x7Fu) << shift;
|
||||
if ((byte & 0x80u) == 0) return true;
|
||||
}
|
||||
failed = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ByteReader::getVarU64(uint64_t& out) {
|
||||
out = 0;
|
||||
for (int shift = 0; shift < 70; shift += 7) {
|
||||
uint8_t byte;
|
||||
if (!getU8(byte)) return false;
|
||||
out |= static_cast<uint64_t>(byte & 0x7Fu) << shift;
|
||||
if ((byte & 0x80u) == 0) return true;
|
||||
}
|
||||
failed = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ByteReader::getVarI32(int32_t& out) {
|
||||
uint32_t raw;
|
||||
if (!getVarU32(raw)) return false;
|
||||
out = zigzagDecode32(raw);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getVarI64(int64_t& out) {
|
||||
uint64_t raw;
|
||||
if (!getVarU64(raw)) return false;
|
||||
out = zigzagDecode64(raw);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getFloat(float& out) {
|
||||
uint32_t bits;
|
||||
if (!getU32(bits)) return false;
|
||||
std::memcpy(&out, &bits, sizeof(out));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getDouble(double& out) {
|
||||
uint64_t bits;
|
||||
if (!getU64(bits)) return false;
|
||||
std::memcpy(&out, &bits, sizeof(out));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getBytes(uint8_t* dst, std::size_t n) {
|
||||
if (!need(n)) return false;
|
||||
std::memcpy(dst, data + cursor, n);
|
||||
cursor += n;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ByteReader::getString(std::string& out) {
|
||||
uint32_t len;
|
||||
if (!getVarU32(len)) return false;
|
||||
if (!need(len)) return false;
|
||||
out.assign(reinterpret_cast<const char*>(data + cursor), len);
|
||||
cursor += len;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t zigzagEncode32(int32_t v) {
|
||||
return (static_cast<uint32_t>(v) << 1) ^ static_cast<uint32_t>(v >> 31);
|
||||
}
|
||||
|
||||
int32_t zigzagDecode32(uint32_t v) {
|
||||
return static_cast<int32_t>((v >> 1) ^ (0u - (v & 1u)));
|
||||
}
|
||||
|
||||
uint64_t zigzagEncode64(int64_t v) {
|
||||
return (static_cast<uint64_t>(v) << 1) ^ static_cast<uint64_t>(v >> 63);
|
||||
}
|
||||
|
||||
int64_t zigzagDecode64(uint64_t v) {
|
||||
return static_cast<int64_t>((v >> 1) ^ (0ull - (v & 1ull)));
|
||||
}
|
||||
|
||||
uint32_t fnv1a32(const uint8_t* data, std::size_t size) {
|
||||
uint32_t h = 2166136261u;
|
||||
for (std::size_t i = 0; i < size; ++i) {
|
||||
h ^= data[i];
|
||||
h *= 16777619u;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
uint64_t fnv1a64(const uint8_t* data, std::size_t size) {
|
||||
uint64_t h = 1469598103934665603ull;
|
||||
for (std::size_t i = 0; i < size; ++i) {
|
||||
h ^= data[i];
|
||||
h *= 1099511628211ull;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
uint32_t adler32(const uint8_t* data, std::size_t size) {
|
||||
const uint32_t modulo = 65521u;
|
||||
uint32_t a = 1, b = 0;
|
||||
for (std::size_t i = 0; i < size; ++i) {
|
||||
a = (a + data[i]) % modulo;
|
||||
b = (b + a) % modulo;
|
||||
}
|
||||
return (b << 16) | a;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr uint8_t rleRunFlag = 0x80;
|
||||
constexpr int rleMinRun = 3;
|
||||
constexpr int rleMaxRun = 130;
|
||||
constexpr int rleMaxLiteral = 128;
|
||||
|
||||
} // namespace
|
||||
|
||||
ByteBuffer rleEncode(const uint8_t* data, std::size_t size) {
|
||||
ByteBuffer out;
|
||||
std::size_t i = 0;
|
||||
while (i < size) {
|
||||
std::size_t runEnd = i + 1;
|
||||
while (runEnd < size && data[runEnd] == data[i] &&
|
||||
runEnd - i < static_cast<std::size_t>(rleMaxRun))
|
||||
++runEnd;
|
||||
std::size_t runLen = runEnd - i;
|
||||
if (runLen >= static_cast<std::size_t>(rleMinRun)) {
|
||||
out.push_back(static_cast<uint8_t>(rleRunFlag | (runLen - rleMinRun)));
|
||||
out.push_back(data[i]);
|
||||
i = runEnd;
|
||||
continue;
|
||||
}
|
||||
std::size_t litStart = i;
|
||||
while (i < size) {
|
||||
std::size_t look = i + 1;
|
||||
while (look < size && data[look] == data[i] &&
|
||||
look - i < static_cast<std::size_t>(rleMinRun))
|
||||
++look;
|
||||
if (look - i >= static_cast<std::size_t>(rleMinRun)) break;
|
||||
++i;
|
||||
if (i - litStart == static_cast<std::size_t>(rleMaxLiteral)) break;
|
||||
}
|
||||
std::size_t litLen = i - litStart;
|
||||
out.push_back(static_cast<uint8_t>(litLen - 1));
|
||||
out.insert(out.end(), data + litStart, data + i);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
bool rleDecode(const uint8_t* data, std::size_t size, ByteBuffer& out) {
|
||||
out.clear();
|
||||
std::size_t i = 0;
|
||||
while (i < size) {
|
||||
uint8_t control = data[i++];
|
||||
if (control & rleRunFlag) {
|
||||
std::size_t runLen = static_cast<std::size_t>(control & 0x7Fu) + rleMinRun;
|
||||
if (i >= size) return false;
|
||||
uint8_t value = data[i++];
|
||||
out.insert(out.end(), runLen, value);
|
||||
} else {
|
||||
std::size_t litLen = static_cast<std::size_t>(control) + 1;
|
||||
if (i + litLen > size) return false;
|
||||
out.insert(out.end(), data + i, data + i + litLen);
|
||||
i += litLen;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int lzMinMatch = 4;
|
||||
constexpr int lzHashBits = 15;
|
||||
constexpr int lzHashSize = 1 << lzHashBits;
|
||||
constexpr int lzMaxChain = 128;
|
||||
constexpr uint32_t lzMaxOffset = 1u << 20;
|
||||
|
||||
uint32_t hash4(const uint8_t* p) {
|
||||
uint32_t v;
|
||||
std::memcpy(&v, p, sizeof(v));
|
||||
return (v * 2654435761u) >> (32 - lzHashBits);
|
||||
}
|
||||
|
||||
void putVarU32Raw(ByteBuffer& out, uint32_t v) {
|
||||
while (v >= 0x80u) {
|
||||
out.push_back(static_cast<uint8_t>(v) | 0x80u);
|
||||
v >>= 7;
|
||||
}
|
||||
out.push_back(static_cast<uint8_t>(v));
|
||||
}
|
||||
|
||||
bool getVarU32Raw(const uint8_t* data, std::size_t size, std::size_t& pos, uint32_t& out) {
|
||||
out = 0;
|
||||
for (int shift = 0; shift < 35; shift += 7) {
|
||||
if (pos >= size) return false;
|
||||
uint8_t byte = data[pos++];
|
||||
out |= static_cast<uint32_t>(byte & 0x7Fu) << shift;
|
||||
if ((byte & 0x80u) == 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
ByteBuffer lz77Compress(const uint8_t* data, std::size_t size) {
|
||||
ByteBuffer out;
|
||||
putVarU32Raw(out, static_cast<uint32_t>(size));
|
||||
if (size == 0) return out;
|
||||
|
||||
std::vector<int32_t> head(lzHashSize, -1);
|
||||
std::vector<int32_t> prev(size, -1);
|
||||
|
||||
std::size_t pos = 0;
|
||||
std::size_t flagIndex = out.size();
|
||||
out.push_back(0);
|
||||
int flagBit = 0;
|
||||
|
||||
auto insert = [&](std::size_t at) {
|
||||
if (at + lzMinMatch <= size) {
|
||||
uint32_t h = hash4(data + at);
|
||||
prev[at] = head[h];
|
||||
head[h] = static_cast<int32_t>(at);
|
||||
}
|
||||
};
|
||||
|
||||
while (pos < size) {
|
||||
std::size_t bestLen = 0;
|
||||
std::size_t bestOffset = 0;
|
||||
if (pos + lzMinMatch <= size) {
|
||||
uint32_t h = hash4(data + pos);
|
||||
int32_t candidate = head[h];
|
||||
int chain = 0;
|
||||
std::size_t maxLen = size - pos;
|
||||
while (candidate >= 0 && chain < lzMaxChain) {
|
||||
std::size_t cand = static_cast<std::size_t>(candidate);
|
||||
if (pos - cand > lzMaxOffset) break;
|
||||
std::size_t len = 0;
|
||||
while (len < maxLen && data[cand + len] == data[pos + len]) ++len;
|
||||
if (len > bestLen) {
|
||||
bestLen = len;
|
||||
bestOffset = pos - cand;
|
||||
if (len == maxLen) break;
|
||||
}
|
||||
candidate = prev[cand];
|
||||
++chain;
|
||||
}
|
||||
}
|
||||
|
||||
if (bestLen >= static_cast<std::size_t>(lzMinMatch)) {
|
||||
out[flagIndex] = static_cast<uint8_t>(out[flagIndex] | (1u << flagBit));
|
||||
putVarU32Raw(out, static_cast<uint32_t>(bestOffset));
|
||||
putVarU32Raw(out, static_cast<uint32_t>(bestLen - lzMinMatch));
|
||||
std::size_t end = pos + bestLen;
|
||||
while (pos < end) {
|
||||
insert(pos);
|
||||
++pos;
|
||||
}
|
||||
} else {
|
||||
out.push_back(data[pos]);
|
||||
insert(pos);
|
||||
++pos;
|
||||
}
|
||||
|
||||
if (++flagBit == 8) {
|
||||
flagBit = 0;
|
||||
if (pos < size) {
|
||||
flagIndex = out.size();
|
||||
out.push_back(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
bool lz77Decompress(const uint8_t* data, std::size_t size, ByteBuffer& out) {
|
||||
out.clear();
|
||||
std::size_t pos = 0;
|
||||
uint32_t rawLen;
|
||||
if (!getVarU32Raw(data, size, pos, rawLen)) return false;
|
||||
out.reserve(rawLen);
|
||||
|
||||
while (out.size() < rawLen) {
|
||||
if (pos >= size) return false;
|
||||
uint8_t flag = data[pos++];
|
||||
for (int bit = 0; bit < 8 && out.size() < rawLen; ++bit) {
|
||||
if (flag & (1u << bit)) {
|
||||
uint32_t offset, extra;
|
||||
if (!getVarU32Raw(data, size, pos, offset)) return false;
|
||||
if (!getVarU32Raw(data, size, pos, extra)) return false;
|
||||
std::size_t len = static_cast<std::size_t>(extra) + lzMinMatch;
|
||||
if (offset == 0 || offset > out.size()) return false;
|
||||
if (out.size() + len > rawLen) return false;
|
||||
std::size_t from = out.size() - offset;
|
||||
for (std::size_t i = 0; i < len; ++i) out.push_back(out[from + i]);
|
||||
} else {
|
||||
if (pos >= size) return false;
|
||||
out.push_back(data[pos++]);
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.size() == rawLen;
|
||||
}
|
||||
|
||||
ByteBuffer compress(const uint8_t* data, std::size_t size) {
|
||||
ByteBuffer rle = rleEncode(data, size);
|
||||
ByteBuffer lz = lz77Compress(data, size);
|
||||
|
||||
CodecMethod method = CodecMethod::Store;
|
||||
const ByteBuffer* payload = nullptr;
|
||||
std::size_t best = size;
|
||||
if (rle.size() < best) {
|
||||
best = rle.size();
|
||||
method = CodecMethod::Rle;
|
||||
payload = &rle;
|
||||
}
|
||||
if (lz.size() < best) {
|
||||
method = CodecMethod::Lz77;
|
||||
payload = &lz;
|
||||
}
|
||||
|
||||
ByteWriter head;
|
||||
head.putU8(static_cast<uint8_t>(method));
|
||||
head.putVarU32(static_cast<uint32_t>(size));
|
||||
head.putU32(fnv1a32(data, size));
|
||||
|
||||
ByteBuffer out = head.bytes();
|
||||
if (method == CodecMethod::Store)
|
||||
out.insert(out.end(), data, data + size);
|
||||
else
|
||||
out.insert(out.end(), payload->begin(), payload->end());
|
||||
return out;
|
||||
}
|
||||
|
||||
ByteBuffer compress(const ByteBuffer& in) { return compress(in.data(), in.size()); }
|
||||
|
||||
bool decompress(const uint8_t* data, std::size_t size, ByteBuffer& out) {
|
||||
ByteReader reader(data, size);
|
||||
uint8_t rawMethod;
|
||||
uint32_t rawLen;
|
||||
uint32_t checksum;
|
||||
if (!reader.getU8(rawMethod)) return false;
|
||||
if (!reader.getVarU32(rawLen)) return false;
|
||||
if (!reader.getU32(checksum)) return false;
|
||||
|
||||
const uint8_t* payload = data + reader.position();
|
||||
std::size_t payloadSize = reader.remaining();
|
||||
|
||||
switch (static_cast<CodecMethod>(rawMethod)) {
|
||||
case CodecMethod::Store:
|
||||
if (payloadSize != rawLen) return false;
|
||||
out.assign(payload, payload + payloadSize);
|
||||
break;
|
||||
case CodecMethod::Rle:
|
||||
if (!rleDecode(payload, payloadSize, out)) return false;
|
||||
break;
|
||||
case CodecMethod::Lz77:
|
||||
if (!lz77Decompress(payload, payloadSize, out)) return false;
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
if (out.size() != rawLen) return false;
|
||||
return fnv1a32(out.data(), out.size()) == checksum;
|
||||
}
|
||||
|
||||
} // namespace vf
|
||||
Reference in new issue
Block a user