fix: MSVC ABI, heap allocator, deferred timer - vanillafixes compat
- Restore MSVC ABI (was accidentally GNU since v0.6.0, broke .CRT section) - Replace game allocator with Windows process heap for filecache and libdeflate malloc/free - game allocator not initialized during DllMain when injected via CreateRemoteThread - Defer timer calibration (Sleep 500ms) to lateInit - blocks under loader lock during DllMain - Remove exported malloc/free symbols from DLL - Eliminate addObject compilation units for SSE files - direct @import with AVX target instead - Heap-allocate filecache (was 9.3MB static BSS) - Strip transform44 of performance/ externs, pure profiling only - Rename performance/ to weirdperformance/ to match module convention - Skip default-off modules in all-variants build step - Remove dead debug vars and stride logging from particle_sse
This commit is contained in:
MarcelineVQ
@@ -0,0 +1,92 @@
|
||||
# libdeflate Integration Research
|
||||
|
||||
## Findings
|
||||
|
||||
### WoW Compression Types
|
||||
|
||||
Dispatch table at `.rdata` 0x80FA1C, 6 entries (mask, function_ptr):
|
||||
|
||||
| Bit | Mask | Address | Function | Type |
|
||||
|-----|------|------------|-----------------------------------|----------------|
|
||||
| 0 | 0x01 | 0x661350 | ProcessCompressionContext | Huffman/sparse |
|
||||
| 1 | 0x02 | 0x660800 | Compression_TryDecompress | **zlib** |
|
||||
| 4 | 0x10 | 0x660C10 | BZip2Decompressor_TryDecompress | bzip2 |
|
||||
| 5 | 0x20 | 0x660AC0 | PKWare_DecompressData | PKWare DCL |
|
||||
| 6 | 0x40 | 0x660F60 | AudioCodec_DecompressMonoSamples | ADPCM mono |
|
||||
| 7 | 0x80 | 0x6611E0 | AudioCodec_DecompressStereoSamples| ADPCM stereo |
|
||||
|
||||
Type byte is a bitmask — multiple bits can be set for chained compression.
|
||||
In practice, **100% of observed calls use type 0x02 (pure zlib, no chaining)**.
|
||||
|
||||
### Data Format
|
||||
|
||||
```
|
||||
[type_byte=0x02] [zlib_stream: 78 9C ...]
|
||||
```
|
||||
|
||||
Standard zlib header `78 9C` = deflate method, 32K window, default compression.
|
||||
|
||||
### Volume (observed during gameplay)
|
||||
|
||||
- Loading: 93K calls, 170MB compressed → 380MB decompressed in 2.9s
|
||||
- Gameplay: ~4-5K calls per 7.5s dump period
|
||||
|
||||
### Function Signatures (assembly-verified)
|
||||
|
||||
```
|
||||
DecompressData_WithOptions (0x661A80):
|
||||
__stdcall(outBuf, &outSize, inBuf, inSize_VALUE, flags) RET 0x14
|
||||
Note: param4 is a VALUE not a pointer. param2 is a POINTER to size.
|
||||
The function modifies *param2 to reflect actual decompressed size.
|
||||
|
||||
Compression_TryDecompress (0x660800):
|
||||
__fastcall(ECX=outBuf, EDX=&outSize, stack=compSize, flags) RET 0x0C
|
||||
|
||||
BZip2Decompressor_Decompress (0x660740):
|
||||
__stdcall(inBuf, &outSize, &uncompSize, flags) RET 0x10
|
||||
|
||||
FreeMemory (0x646430):
|
||||
__stdcall(ptr, filename, line, flags) RET 0x10 ← 4 params NOT 3!
|
||||
|
||||
ReallocMemory (0x646320):
|
||||
__stdcall(ptr, size, filename, line, flags) RET 0x14
|
||||
When ptr=NULL, acts as malloc via AllocateBufferWithPowerOfTwo.
|
||||
```
|
||||
|
||||
### libdeflate Status
|
||||
|
||||
- **Compiled**: static lib, x86-windows-gnu target, linked into msvc DLL
|
||||
- **Sanity test PASSES**: decompresses "hello" correctly
|
||||
- **CPU features**: 0x8000001F = SSE2 + SSSE3 + SSE4.1 + BMI1 + BMI2 (all valid under Wine)
|
||||
- **AVX-512 disabled** at compile time (not available on 32-bit x86)
|
||||
|
||||
### Critical Bug Found
|
||||
|
||||
The original `DecompressData_WithOptions` **modifies the input buffer** during
|
||||
decompression (overlap handling copies data around). When our hook calls the
|
||||
original first then tries to run libdeflate on the same input, the data has
|
||||
been corrupted. Fix: save input data before calling original.
|
||||
|
||||
### Other Bugs Found During Integration
|
||||
|
||||
1. **FreeMemory param count**: RET 0x10 = 4 params, was declared with 3 → stack corruption
|
||||
2. **Stack overflow**: 64KB stack buffer for output → replaced with heap/static allocation
|
||||
3. **param4 is VALUE not pointer**: `inSize` is passed by value, not `&inSize`
|
||||
|
||||
### Hook Architecture
|
||||
|
||||
```
|
||||
DecompressData_WithOptions hook:
|
||||
1. Save input buffer (original modifies it)
|
||||
2. Call original → time it
|
||||
3. If type==0x02 (pure zlib):
|
||||
a. Run libdeflate_zlib_decompress on saved input
|
||||
b. Time it, compare results
|
||||
4. Return original's result
|
||||
```
|
||||
|
||||
### Next Steps
|
||||
|
||||
- Test with saved input buffer fix
|
||||
- If timing comparison works, implement replacement mode (skip original, use libdeflate only)
|
||||
- Expected speedup: ~2x based on libdeflate benchmarks (generic C path on 32-bit)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,376 @@
|
||||
//! SSE-optimized collision math — compiled ReleaseFast even in Debug builds.
|
||||
//!
|
||||
//! Functions are exported and called via `extern fn` from the Debug transform44 module.
|
||||
//! This separation is required because Zig module imports inherit the parent's optimization
|
||||
//! level, so only a separate `addObject(.{ .optimize = .ReleaseFast })` gets real SSE codegen.
|
||||
|
||||
const V4 = @Vector(4, f32);
|
||||
|
||||
const CLIP_EPSILON: f32 = @bitCast(@as(u32, 0x3ab60b61)); // +0.00139, global at 0x80dfec
|
||||
const MOVEMENT_EPSILON: f32 = @bitCast(@as(u32, 0x35800000)); // 9.54e-7, global at 0x8026bc
|
||||
|
||||
pub fn clipPolygonToSinglePlane(plane_addr: u32, poly_addr: u32, attrib_bits: u32) void {
|
||||
const plane: [*]const f32 = @ptrFromInt(plane_addr);
|
||||
const poly: [*]f32 = @ptrFromInt(poly_addr);
|
||||
const new_attrib: f32 = @bitCast(attrib_bits);
|
||||
|
||||
// Vertex count at poly+0xF0 (byte offset 240, float offset 60)
|
||||
const count_ptr: *align(1) u32 = @ptrCast(poly + 60);
|
||||
const n = count_ptr.*;
|
||||
if (n == 0) return;
|
||||
|
||||
// Load plane as 4-wide vector for SSE dot product
|
||||
const pv: @Vector(4, f32) = .{ plane[0], plane[1], plane[2], plane[3] };
|
||||
|
||||
// --- Phase 1: Compute signed distances ---
|
||||
var dists: [15]f32 = undefined;
|
||||
var min_dist: f32 = @bitCast(@as(u32, 0x7f7fffff)); // FLT_MAX
|
||||
var max_dist: f32 = @bitCast(@as(u32, 0xff7fffff)); // -FLT_MAX
|
||||
|
||||
var i: u32 = 0;
|
||||
while (i < n) : (i += 1) {
|
||||
const b = i * 3;
|
||||
const v: @Vector(4, f32) = .{ poly[b], poly[b + 1], poly[b + 2], 1.0 };
|
||||
const d = -@reduce(.Add, v * pv);
|
||||
dists[i] = d;
|
||||
if (d < min_dist) min_dist = d;
|
||||
if (d > max_dist) max_dist = d;
|
||||
}
|
||||
|
||||
// --- Phase 2: Early exits (exact same thresholds as original) ---
|
||||
if (min_dist > -CLIP_EPSILON) return; // all inside
|
||||
if (max_dist < CLIP_EPSILON) {
|
||||
count_ptr.* = 0; // all outside
|
||||
return;
|
||||
}
|
||||
|
||||
// --- Phase 3: Copy polygon to local buffers ---
|
||||
const attribs: [*]f32 = poly + 45; // +0xB4
|
||||
var tmp_v: [15 * 3]f32 = undefined;
|
||||
var tmp_a: [15]f32 = undefined;
|
||||
for (0..n * 3) |j| tmp_v[j] = poly[j];
|
||||
for (0..n) |j| tmp_a[j] = attribs[j];
|
||||
|
||||
// --- Phase 4: Sutherland-Hodgman edge walk ---
|
||||
var out: u32 = 0;
|
||||
var prev: u32 = n - 1;
|
||||
var pd = dists[prev];
|
||||
|
||||
i = 0;
|
||||
while (i < n) : (i += 1) {
|
||||
const cd = dists[i];
|
||||
|
||||
if (pd >= 0.0) {
|
||||
if (cd >= 0.0) {
|
||||
emit(poly, attribs, &out, (tmp_v[i * 3 ..]).ptr, tmp_a[i]);
|
||||
} else {
|
||||
if (pd > CLIP_EPSILON) {
|
||||
lerp(poly, attribs, &out, (tmp_v[prev * 3 ..]).ptr, (tmp_v[i * 3 ..]).ptr, pd / (cd - pd), new_attrib);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (cd >= 0.0) {
|
||||
if (cd > CLIP_EPSILON) {
|
||||
lerp(poly, attribs, &out, (tmp_v[prev * 3 ..]).ptr, (tmp_v[i * 3 ..]).ptr, pd / (cd - pd), new_attrib);
|
||||
}
|
||||
emit(poly, attribs, &out, (tmp_v[i * 3 ..]).ptr, tmp_a[i]);
|
||||
}
|
||||
}
|
||||
|
||||
prev = i;
|
||||
pd = cd;
|
||||
}
|
||||
|
||||
count_ptr.* = if (out >= 3) out else 0;
|
||||
}
|
||||
|
||||
inline fn emit(poly: [*]f32, attribs: [*]f32, out: *u32, v: [*]const f32, a: f32) void {
|
||||
const o = out.*;
|
||||
const b = o * 3;
|
||||
poly[b] = v[0];
|
||||
poly[b + 1] = v[1];
|
||||
poly[b + 2] = v[2];
|
||||
attribs[o] = a;
|
||||
out.* = o + 1;
|
||||
}
|
||||
|
||||
inline fn lerp(poly: [*]f32, attribs: [*]f32, out: *u32, p: [*]const f32, c: [*]const f32, t: f32, a: f32) void {
|
||||
const o = out.*;
|
||||
const b = o * 3;
|
||||
// intersection = prev - (curr - prev) * t
|
||||
poly[b] = p[0] - (c[0] - p[0]) * t;
|
||||
poly[b + 1] = p[1] - (c[1] - p[1]) * t;
|
||||
poly[b + 2] = p[2] - (c[2] - p[2]) * t;
|
||||
attribs[o] = a;
|
||||
out.* = o + 1;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// BuildTrianglePlanes (0x632460)
|
||||
// __fastcall(ECX=vertices, EDX=triangle_indices(byte*),
|
||||
// stack: plane_normal*, offset_vector*, output_planes*)
|
||||
// Returns: int (1=ok, 0=degenerate)
|
||||
//
|
||||
// Builds 4 clipping planes from a triangle + offset:
|
||||
// planes[0..2]: edge planes (perpendicular to triangle, one per edge)
|
||||
// planes[3]: cap plane (from plane_normal, offset by offset_vector)
|
||||
// =============================================================================
|
||||
|
||||
pub fn buildTrianglePlanes(verts_addr: u32, indices_addr: u32, normal_addr: u32, offset_addr: u32, out_addr: u32) u32 {
|
||||
const verts: [*]const f32 = @ptrFromInt(verts_addr);
|
||||
const indices: [*]const u8 = @ptrFromInt(indices_addr);
|
||||
const plane_normal: [*]const f32 = @ptrFromInt(normal_addr);
|
||||
const offset_vec: [*]const f32 = @ptrFromInt(offset_addr);
|
||||
const output: [*]f32 = @ptrFromInt(out_addr);
|
||||
|
||||
const ofs = loadV3(offset_vec);
|
||||
|
||||
// Load 3 triangle vertices
|
||||
var tv: [3]V4 = undefined;
|
||||
for (0..3) |i| {
|
||||
const idx = @as(u32, indices[i]) * 3;
|
||||
tv[i] = loadV3(verts + idx);
|
||||
}
|
||||
|
||||
// Edge tables: for edge i, v0=i, v1=next, v2=opposite
|
||||
const next = [3]u8{ 1, 2, 0 };
|
||||
const opp = [3]u8{ 2, 0, 1 };
|
||||
|
||||
// Build 3 edge planes
|
||||
for (0..3) |i| {
|
||||
const v0 = tv[i];
|
||||
const v1 = tv[next[i]];
|
||||
const v2 = tv[opp[i]];
|
||||
const offset_v = v0 + ofs; // offset vertex
|
||||
|
||||
// Check for degenerate triangle: cross(offset_vec, v1 - v0)
|
||||
const edge = v1 - v0;
|
||||
const check_normal = cross3(ofs, edge);
|
||||
if (dot3(check_normal, check_normal) < MOVEMENT_EPSILON) return 0;
|
||||
|
||||
// Calculate plane from 3 points (v0, v1, offset_v)
|
||||
const e1 = offset_v - v0; // = offset_vec
|
||||
const e2 = v1 - v0; // = edge
|
||||
var normal = cross3(e2, e1);
|
||||
const len_sq = dot3(normal, normal);
|
||||
const inv_len: V4 = @splat(1.0 / @sqrt(len_sq));
|
||||
normal = normal * inv_len;
|
||||
const d = -dot3(normal, v0);
|
||||
|
||||
// Check orientation: if opposite vertex is on positive side, negate
|
||||
const to_v2 = v2 - v0;
|
||||
const plane_idx = i * 4;
|
||||
if (dot3(to_v2, normal) > 0.0) {
|
||||
output[plane_idx + 0] = -normal[0];
|
||||
output[plane_idx + 1] = -normal[1];
|
||||
output[plane_idx + 2] = -normal[2];
|
||||
output[plane_idx + 3] = -d;
|
||||
} else {
|
||||
output[plane_idx + 0] = normal[0];
|
||||
output[plane_idx + 1] = normal[1];
|
||||
output[plane_idx + 2] = normal[2];
|
||||
output[plane_idx + 3] = d;
|
||||
}
|
||||
}
|
||||
|
||||
// 4th plane: cap plane from plane_normal
|
||||
const pn = loadV3(plane_normal);
|
||||
const cap_point = tv[0] + ofs;
|
||||
output[12] = plane_normal[0];
|
||||
output[13] = plane_normal[1];
|
||||
output[14] = plane_normal[2];
|
||||
output[15] = -dot3(pn, cap_point);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// rayTriangleIntersection (0x7c29f0)
|
||||
// Möller-Trumbore ray-triangle intersection test
|
||||
// __fastcall(ECX=ray[6], EDX=verts_base, stack: indices_u16[3], out_dist*, out_bary*, tolerance)
|
||||
// Returns: 1 = hit, 0 = miss
|
||||
// =============================================================================
|
||||
|
||||
pub fn rayTriangleIntersection(
|
||||
ray_addr: u32,
|
||||
verts_addr: u32,
|
||||
indices_addr: u32,
|
||||
out_dist_addr: u32,
|
||||
out_bary_addr: u32,
|
||||
tolerance_bits: u32,
|
||||
) u32 {
|
||||
const ray: [*]const f32 = @ptrFromInt(ray_addr);
|
||||
const verts: [*]const f32 = @ptrFromInt(verts_addr);
|
||||
const indices: [*]const u16 = @ptrFromInt(indices_addr);
|
||||
const tolerance: f32 = @bitCast(tolerance_bits);
|
||||
|
||||
const neg_tol = -tolerance;
|
||||
const one_plus_tol = 1.0 + tolerance;
|
||||
|
||||
// Load vertex positions via indices (each vertex = 3 floats, stride = 12 bytes)
|
||||
const idx0: u32 = @as(u32, indices[0]) * 3;
|
||||
const idx1: u32 = @as(u32, indices[1]) * 3;
|
||||
const idx2: u32 = @as(u32, indices[2]) * 3;
|
||||
|
||||
const v0 = loadV3(verts + idx0);
|
||||
const v1 = loadV3(verts + idx1);
|
||||
const v2 = loadV3(verts + idx2);
|
||||
|
||||
// Ray: origin at ray[0..3], direction at ray[3..6]
|
||||
const origin = loadV3(ray);
|
||||
const dir = loadV3(ray + 3);
|
||||
|
||||
// Möller-Trumbore algorithm
|
||||
const edge1 = v1 - v0;
|
||||
const edge2 = v2 - v0;
|
||||
const h = cross3(dir, edge2);
|
||||
const det = dot3(edge1, h);
|
||||
|
||||
// Determinant thresholds from WoW binary (both 0.0 at compile time = two-sided test)
|
||||
const det_neg = @as(*const f32, @ptrFromInt(0x0081d9bc)).*;
|
||||
const det_pos = @as(*const f32, @ptrFromInt(0x0080e2e4)).*;
|
||||
|
||||
if (det > det_neg and det < det_pos) return 0;
|
||||
|
||||
const inv_det = 1.0 / det;
|
||||
|
||||
// Barycentric u
|
||||
const s = origin - v0;
|
||||
const u_val = dot3(s, h) * inv_det;
|
||||
if (u_val < neg_tol or u_val > one_plus_tol) return 0;
|
||||
|
||||
// Barycentric v
|
||||
const q = cross3(s, edge1);
|
||||
const v_val = dot3(dir, q) * inv_det;
|
||||
if (v_val < neg_tol or (u_val + v_val) > one_plus_tol) return 0;
|
||||
|
||||
// Intersection distance t
|
||||
const t = dot3(edge2, q) * inv_det;
|
||||
|
||||
if (out_dist_addr != 0) {
|
||||
@as(*f32, @ptrFromInt(out_dist_addr)).* = t;
|
||||
}
|
||||
if (out_bary_addr != 0) {
|
||||
const bary: [*]f32 = @ptrFromInt(out_bary_addr);
|
||||
bary[0] = u_val;
|
||||
bary[1] = v_val;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// multiplyMatrix4x4 (0x7bc6a0)
|
||||
// SSE replacement for 542 bytes of x87 FPU matrix multiply.
|
||||
// __fastcall(ECX=result, EDX=left, stack: right), RET 0x4
|
||||
// Row-major: result[i][j] = sum(left[i][k] * right[k][j], k=0..3)
|
||||
// Returns result pointer (EAX = result_addr).
|
||||
// =============================================================================
|
||||
|
||||
pub fn multiplyMatrix4x4(result_addr: u32, left_addr: u32, right_addr: u32) u32 {
|
||||
const result: [*]f32 = @ptrFromInt(result_addr);
|
||||
const left: [*]const f32 = @ptrFromInt(left_addr);
|
||||
const right: [*]const f32 = @ptrFromInt(right_addr);
|
||||
|
||||
// Load all 4 rows of right matrix
|
||||
const r0: V4 = .{ right[0], right[1], right[2], right[3] };
|
||||
const r1: V4 = .{ right[4], right[5], right[6], right[7] };
|
||||
const r2: V4 = .{ right[8], right[9], right[10], right[11] };
|
||||
const r3: V4 = .{ right[12], right[13], right[14], right[15] };
|
||||
|
||||
// For each row of left, broadcast-multiply-add against right rows
|
||||
inline for (0..4) |i| {
|
||||
const b = i * 4;
|
||||
const out = splat4(left[b]) * r0 + splat4(left[b + 1]) * r1 + splat4(left[b + 2]) * r2 + splat4(left[b + 3]) * r3;
|
||||
storeV4(result + b, out);
|
||||
}
|
||||
|
||||
return result_addr;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// rotateMatrixByAxisAngle (0x7bdd60)
|
||||
// Builds axis-angle rotation matrix (Rodrigues) then multiplies.
|
||||
// Replaces 853 bytes of x87 FPU (createAxisAngleRotationMatrix 311B +
|
||||
// multiplyMatrix4x4 542B) with SSE @Vector math.
|
||||
// __thiscall(ECX=matrix, stack: angle, axis_ptr, is_unit_flag)
|
||||
// =============================================================================
|
||||
|
||||
pub fn rotateMatrixByAxisAngle(
|
||||
matrix_addr: u32,
|
||||
angle_bits: u32,
|
||||
axis_addr: u32,
|
||||
is_unit: u32,
|
||||
) void {
|
||||
const matrix: [*]f32 = @ptrFromInt(matrix_addr);
|
||||
const angle: f32 = @bitCast(angle_bits);
|
||||
const axis_ptr: [*]const f32 = @ptrFromInt(axis_addr);
|
||||
|
||||
// Load and optionally normalize axis
|
||||
var ax = axis_ptr[0];
|
||||
var ay = axis_ptr[1];
|
||||
var az = axis_ptr[2];
|
||||
|
||||
if (is_unit == 0) {
|
||||
const inv_len = 1.0 / @sqrt(ax * ax + ay * ay + az * az);
|
||||
ax *= inv_len;
|
||||
ay *= inv_len;
|
||||
az *= inv_len;
|
||||
}
|
||||
|
||||
// Rodrigues rotation matrix (row-major, matching WoW's layout)
|
||||
const c = cosf(angle);
|
||||
const s = sinf(angle);
|
||||
const t = 1.0 - c;
|
||||
|
||||
// Build rotation matrix on stack, then multiply: result = rot * input
|
||||
// Row 3 of rotation is {0,0,0,1} so result row 3 = input row 3 (identity).
|
||||
// We build the full 4x4 rot matrix and use multiplyMatrix4x4 for the multiply.
|
||||
var rot: [16]f32 = .{
|
||||
ax * ax * t + c, ax * ay * t + az * s, ax * az * t - ay * s, 0,
|
||||
ax * ay * t - az * s, ay * ay * t + c, ay * az * t + ax * s, 0,
|
||||
ax * az * t + ay * s, ay * az * t - ax * s, az * az * t + c, 0,
|
||||
0, 0, 0, 1,
|
||||
};
|
||||
|
||||
// result = rot * matrix, but result == matrix so use temp to avoid aliasing
|
||||
var tmp: [16]f32 = undefined;
|
||||
_ = multiplyMatrix4x4(@intFromPtr(&tmp), @intFromPtr(&rot), matrix_addr);
|
||||
matrix[0..16].* = tmp;
|
||||
}
|
||||
|
||||
// MSVC CRT sin/cos — linked from the WoW process
|
||||
extern fn sinf(f32) f32;
|
||||
extern fn cosf(f32) f32;
|
||||
|
||||
// =============================================================================
|
||||
// Vector helpers
|
||||
// =============================================================================
|
||||
|
||||
inline fn loadV3(p: [*]const f32) V4 {
|
||||
return .{ p[0], p[1], p[2], 0.0 };
|
||||
}
|
||||
|
||||
inline fn dot3(a: V4, b: V4) f32 {
|
||||
const p = a * b;
|
||||
return p[0] + p[1] + p[2];
|
||||
}
|
||||
|
||||
inline fn cross3(a: V4, b: V4) V4 {
|
||||
const a_yzx = @shuffle(f32, a, undefined, [4]i32{ 1, 2, 0, 3 });
|
||||
const b_zxy = @shuffle(f32, b, undefined, [4]i32{ 2, 0, 1, 3 });
|
||||
const a_zxy = @shuffle(f32, a, undefined, [4]i32{ 2, 0, 1, 3 });
|
||||
const b_yzx = @shuffle(f32, b, undefined, [4]i32{ 1, 2, 0, 3 });
|
||||
return a_yzx * b_zxy - a_zxy * b_yzx;
|
||||
}
|
||||
|
||||
inline fn splat4(v: f32) V4 {
|
||||
return @splat(v);
|
||||
}
|
||||
|
||||
inline fn storeV4(dst: [*]f32, v: V4) void {
|
||||
dst[0] = v[0];
|
||||
dst[1] = v[1];
|
||||
dst[2] = v[2];
|
||||
dst[3] = v[3];
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
//! SSE-optimized spatial culling — compiled ReleaseFast even in Debug builds.
|
||||
//!
|
||||
//! Reimplements PerformSpatialCulling (0x6B8C60) and performCollisionDetection
|
||||
//! (0x6B88E0). Both are leaf functions in the KD-tree traversal that compute
|
||||
//! per-vertex 6-bit outcodes against an AABB, then iterate triangles.
|
||||
//!
|
||||
//! The hot path is the vertex outcode loop: 6 float comparisons per vertex
|
||||
//! (~150 vertices typical). SSE compares all 3 axes in parallel.
|
||||
|
||||
// =============================================================================
|
||||
// Benchmark-only: pure outcode computation, no game function dependencies.
|
||||
// Called from src/bench/main.zig with synthetic vertex data.
|
||||
// =============================================================================
|
||||
|
||||
/// Compute outcodes for `count` vertices at `verts_ptr` (stride 12 bytes = 3 floats)
|
||||
/// against AABB at `bounds_ptr` (6 floats: minX, minY, minZ, maxX, maxY, maxZ).
|
||||
/// Writes results to `out_ptr` (1 byte per vertex).
|
||||
pub fn benchComputeOutcodes(verts_ptr: u32, bounds_ptr: u32, out_ptr: u32, count: u32) void {
|
||||
if (count == 0) return;
|
||||
const v_min: V4 = .{ readF32(bounds_ptr), readF32(bounds_ptr + 4), readF32(bounds_ptr + 8), 0 };
|
||||
const v_max: V4 = .{ readF32(bounds_ptr + 12), readF32(bounds_ptr + 16), readF32(bounds_ptr + 20), 0 };
|
||||
const below_w: @Vector(4, u32) = .{ 0x20, 0x08, 0x02, 0x00 };
|
||||
const above_w: @Vector(4, u32) = .{ 0x10, 0x04, 0x01, 0x00 };
|
||||
|
||||
const out: [*]u8 = @ptrFromInt(out_ptr);
|
||||
var vp: [*]const f32 = @ptrFromInt(verts_ptr);
|
||||
var i: u32 = 0;
|
||||
while (i < count) : (i += 1) {
|
||||
const v: V4 = @as(*align(1) const V4, @ptrCast(vp)).*;
|
||||
const zero: @Vector(4, u32) = @splat(0);
|
||||
const combined = @select(u32, v < v_min, below_w, zero) | @select(u32, v >= v_max, above_w, zero);
|
||||
out[i] = @truncate(combined[0] | combined[1] | combined[2] | combined[3]);
|
||||
vp += 3;
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// FindObjectByGUID (0x464890) cache
|
||||
// stdcall(guidLow, guidHigh) -> objectPtr. RET 0x8.
|
||||
// Direct-mapped cache: hash the 64-bit GUID, check cached result still valid.
|
||||
// =============================================================================
|
||||
|
||||
const GUID_CACHE_BITS = 8;
|
||||
const GUID_CACHE_SIZE = 1 << GUID_CACHE_BITS;
|
||||
const GUID_CACHE_MASK = GUID_CACHE_SIZE - 1;
|
||||
|
||||
const GuidCacheEntry = struct {
|
||||
guid_lo: u32 = 0,
|
||||
guid_hi: u32 = 0,
|
||||
result: u32 = 0,
|
||||
};
|
||||
|
||||
var guid_cache: [GUID_CACHE_SIZE]GuidCacheEntry = [_]GuidCacheEntry{.{}} ** GUID_CACHE_SIZE;
|
||||
|
||||
const origFindObjectByGUID = @as(*const fn (u32, u32) callconv(.{ .x86_stdcall = .{} }) u32, @ptrFromInt(0x464890));
|
||||
|
||||
pub fn findObjectByGUID_Cached(guid_lo: u32, guid_hi: u32) callconv(.{ .x86_stdcall = .{} }) u32 {
|
||||
const hash = (guid_lo ^ (guid_hi *% 0x9E3779B9)) & GUID_CACHE_MASK;
|
||||
const entry = &guid_cache[hash];
|
||||
|
||||
if (entry.guid_lo == guid_lo and entry.guid_hi == guid_hi and entry.result != 0) {
|
||||
// Validate: object at cached address still has this GUID
|
||||
const obj = entry.result;
|
||||
if (readU32(obj + 0x30) == guid_lo and readU32(obj + 0x34) == guid_hi) {
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
// Cache miss or stale: call original
|
||||
const result = @call(.never_tail, origFindObjectByGUID, .{ guid_lo, guid_hi });
|
||||
|
||||
// Store in cache (even if result is 0 -- avoids repeated misses for deleted objects)
|
||||
entry.* = .{ .guid_lo = guid_lo, .guid_hi = guid_hi, .result = result };
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// AddToSpatialGrid (0x6816F0)
|
||||
// __fastcall(ECX=objectPtr), RET
|
||||
// Computes grid bucket index via dot product + scale + round, then
|
||||
// inserts object into the bucket's linked list.
|
||||
// =============================================================================
|
||||
|
||||
// CalculateLinkedListOffset: thiscall(ECX=node, stack=direction) -> ptr
|
||||
const CalculateLinkedListOffset = @as(*const fn (u32, i32) callconv(.{ .x86_thiscall = .{} }) u32, @ptrFromInt(0x6876B0));
|
||||
|
||||
// Grid globals
|
||||
const g_grid_base: u32 = 0xC7BD40; // grid array base (stride 0x6C per bucket)
|
||||
const g_grid_scale: *const f32 = @ptrFromInt(0x810174);
|
||||
const g_grid_offset: *const f32 = @ptrFromInt(0x86861C);
|
||||
// Dot product coefficients at 0xC7CFB8..C7CFC4 (same as entpos view coeffs but different address)
|
||||
const g_spatial_coeffs: u32 = 0xC7CFB8;
|
||||
|
||||
pub fn addToSpatialGridSSE(obj: u32) callconv(.{ .x86_fastcall = .{} }) void {
|
||||
// Dot product: coeff_a * obj[0x5C] + coeff_b * obj[0x60] + coeff_c * obj[0x64] + coeff_d
|
||||
const depth = readF32(g_spatial_coeffs) * readF32(obj + 0x5C) +
|
||||
readF32(g_spatial_coeffs + 4) * readF32(obj + 0x60) +
|
||||
readF32(g_spatial_coeffs + 8) * readF32(obj + 0x64) +
|
||||
readF32(g_spatial_coeffs + 12) - readF32(obj + 0x68);
|
||||
|
||||
// Grid index: round(depth * scale - offset), clamped to [0, 31]
|
||||
const scaled = depth * g_grid_scale.* - g_grid_offset.*;
|
||||
|
||||
// Round to nearest (matches x87 FISTP with default rounding mode)
|
||||
// @round returns f32, then convert to int
|
||||
const rounded = @round(scaled);
|
||||
var idx: i32 = @intFromFloat(rounded);
|
||||
|
||||
if (idx < 0) {
|
||||
idx = 0;
|
||||
} else if (idx >= 0x20) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Bucket layout: grid_base + idx * 0x6C
|
||||
// Bucket+0x18 = offset to node within object
|
||||
// Bucket+0x1C = list head pointer
|
||||
const bucket = g_grid_base + @as(u32, @bitCast(idx)) * 0x6C;
|
||||
const node_offset = readU32(bucket + 0x18);
|
||||
const node = node_offset + obj;
|
||||
|
||||
// If already in a list, unlink first
|
||||
if (readU32(node) != 0) {
|
||||
const prev = @call(.never_tail, CalculateLinkedListOffset, .{ node, -1 });
|
||||
@as(*align(1) u32, @ptrFromInt(prev)).* = readU32(node);
|
||||
@as(*align(1) u32, @ptrFromInt(readU32(node) + 4)).* = readU32(node + 4);
|
||||
@as(*align(1) u32, @ptrFromInt(node)).* = 0;
|
||||
@as(*align(1) u32, @ptrFromInt(node + 4)).* = 0;
|
||||
}
|
||||
|
||||
// Insert at head of bucket list
|
||||
const head = readU32(bucket + 0x1C);
|
||||
@as(*align(1) u32, @ptrFromInt(node)).* = head;
|
||||
@as(*align(1) u32, @ptrFromInt(node + 4)).* = readU32(head + 4);
|
||||
@as(*align(1) u32, @ptrFromInt(head + 4)).* = obj;
|
||||
@as(*align(1) u32, @ptrFromInt(bucket + 0x1C)).* = node;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// ray_triangle_intersection_indexed_int (0x7C2C40)
|
||||
// Same Moller-Trumbore as _indexed_ushort but indices are int* not u16*.
|
||||
// fastcall(ECX=ray, EDX=vertPool, stack: indices, tOut, normalOut, epsilon)
|
||||
// RET 0x10
|
||||
// =============================================================================
|
||||
|
||||
pub fn rayTriIntersectIndexedInt(
|
||||
ray_ptr: u32,
|
||||
vert_pool: u32,
|
||||
indices_ptr: u32,
|
||||
t_out: u32,
|
||||
normal_out: u32,
|
||||
epsilon_bits: u32,
|
||||
) callconv(.{ .x86_fastcall = .{} }) u8 {
|
||||
const epsilon: f32 = @bitCast(epsilon_bits);
|
||||
const neg_eps = -epsilon;
|
||||
const one_plus_eps = 1.0 + epsilon;
|
||||
|
||||
// Indices are int (4 bytes each), not u16
|
||||
const idx0: u32 = @bitCast(readI32(indices_ptr));
|
||||
const idx1: u32 = @bitCast(readI32(indices_ptr + 4));
|
||||
const idx2: u32 = @bitCast(readI32(indices_ptr + 8));
|
||||
|
||||
const v0 = loadVec3(vert_pool + idx0 * 12);
|
||||
const v1 = loadVec3(vert_pool + idx1 * 12);
|
||||
const v2 = loadVec3(vert_pool + idx2 * 12);
|
||||
const ray_origin = loadVec3(ray_ptr);
|
||||
const ray_dir = loadVec3(ray_ptr + 12);
|
||||
|
||||
const edge1 = v1 - v0;
|
||||
const edge2 = v2 - v0;
|
||||
const pvec = cross(ray_dir, edge2);
|
||||
const det = dot3(edge1, pvec);
|
||||
|
||||
if (det > -1e-6 and det < 1e-6) return 0;
|
||||
|
||||
const inv_det = 1.0 / det;
|
||||
const tvec = ray_origin - v0;
|
||||
|
||||
const u = dot3(tvec, pvec) * inv_det;
|
||||
if (u < neg_eps or u > one_plus_eps) return 0;
|
||||
|
||||
const qvec = cross(tvec, edge1);
|
||||
const v = dot3(ray_dir, qvec) * inv_det;
|
||||
if (v < neg_eps or (u + v) > one_plus_eps) return 0;
|
||||
|
||||
if (t_out != 0) {
|
||||
@as(*align(1) f32, @ptrFromInt(t_out)).* = dot3(edge2, qvec) * inv_det;
|
||||
}
|
||||
if (normal_out != 0) {
|
||||
@as(*align(1) f32, @ptrFromInt(normal_out)).* = u;
|
||||
@as(*align(1) f32, @ptrFromInt(normal_out + 4)).* = v;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
fn readI32(addr: u32) i32 {
|
||||
return @as(*align(1) const i32, @ptrFromInt(addr)).*;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Game hook exports
|
||||
// =============================================================================
|
||||
|
||||
// External game functions (resolved at link time via absolute address)
|
||||
// FindOrCreateHashEntry: thiscall(ECX=hashTable from global 0xCA03E4, stack: 5 args) RET 0x14
|
||||
const FindOrCreateHashEntry = @as(*const fn (u32, u32, u32, u32, u32, u32) callconv(.{ .x86_thiscall = .{} }) u32, @ptrFromInt(0x693D60));
|
||||
// Global state used by the game's rendering pipeline
|
||||
const g_visible_count: *u32 = @ptrFromInt(0xCE26E0); // PTR_00ce26e0
|
||||
const g_visible_list: [*]u16 = @ptrFromInt(0xCE26E8); // DAT_00ce26e8
|
||||
const g_render_count: *u32 = @ptrFromInt(0xCE66FC); // PTR_00ce66fc
|
||||
const g_render_list: [*]u16 = @ptrFromInt(0xCDE648); // DAT_00cde648
|
||||
const g_guard: *const u32 = @ptrFromInt(0xCA03E4); // PTR_00ca03e4
|
||||
|
||||
const V4 = @Vector(4, f32);
|
||||
|
||||
fn readU32(addr: u32) u32 {
|
||||
return @as(*align(1) const u32, @ptrFromInt(addr)).*;
|
||||
}
|
||||
fn readU16(addr: u32) u16 {
|
||||
return @as(*align(1) const u16, @ptrFromInt(addr)).*;
|
||||
}
|
||||
fn readF32(addr: u32) f32 {
|
||||
return @as(*align(1) const f32, @ptrFromInt(addr)).*;
|
||||
}
|
||||
|
||||
/// Compute outcodes for all vertices in the mesh using SSE.
|
||||
///
|
||||
/// Per-vertex 6-bit outcode against AABB. Two SIMD compares (v < min, v >= max)
|
||||
/// produce all 6 bits from movemask results. Processes xyz in parallel.
|
||||
///
|
||||
/// Bit layout: 0x20=below_minX, 0x10=above_maxX, 0x08=below_minY,
|
||||
/// 0x04=above_maxY, 0x02=below_minZ, 0x01=above_maxZ
|
||||
fn computeAllOutcodes(
|
||||
hash_entry: u32,
|
||||
min_x: f32,
|
||||
max_x: f32,
|
||||
min_y: f32,
|
||||
max_y: f32,
|
||||
min_z: f32,
|
||||
max_z: f32,
|
||||
cull_flags: *[452]u8,
|
||||
) u32 {
|
||||
const vert_count: u32 = readU16(hash_entry + 6);
|
||||
if (vert_count == 0) return 0;
|
||||
|
||||
const v_min: V4 = .{ min_x, min_y, min_z, 0 };
|
||||
const v_max: V4 = .{ max_x, max_y, max_z, 0 };
|
||||
|
||||
// Bit weights for branchless outcode: below gives 0x20/0x08/0x02, above gives 0x10/0x04/0x01
|
||||
const below_w: @Vector(4, u32) = .{ 0x20, 0x08, 0x02, 0x00 };
|
||||
const above_w: @Vector(4, u32) = .{ 0x10, 0x04, 0x01, 0x00 };
|
||||
|
||||
var vert_ptr: [*]const f32 = @ptrFromInt(hash_entry + 8);
|
||||
var i: u32 = 0;
|
||||
while (i < vert_count) : (i += 1) {
|
||||
// Single 16-byte unaligned load. 4th float is junk from next vertex
|
||||
// but v_min[3]=0, v_max[3]=0, so comparisons on lane 3 produce
|
||||
// below=false (0>=0), above=true (0>=0) -- weight is 0x00 so harmless.
|
||||
const v: V4 = @as(*align(1) const V4, @ptrCast(vert_ptr)).*;
|
||||
|
||||
// Bool vectors -> u32 vectors (0 or 0xFFFFFFFF), AND with weights, horizontal OR
|
||||
const zero: @Vector(4, u32) = @splat(0);
|
||||
const below_masked = @select(u32, v < v_min, below_w, zero);
|
||||
const above_masked = @select(u32, v >= v_max, above_w, zero);
|
||||
const combined = below_masked | above_masked;
|
||||
|
||||
// Horizontal OR of 4 lanes -> single outcode byte
|
||||
cull_flags[i] = @truncate(combined[0] | combined[1] | combined[2] | combined[3]);
|
||||
|
||||
vert_ptr += 3; // stride 12 bytes = 3 floats
|
||||
}
|
||||
return vert_count;
|
||||
}
|
||||
|
||||
/// PerformSpatialCulling (0x6B8C60)
|
||||
/// __thiscall(this, keyData, keySize) -> u32. RET 0x8.
|
||||
///
|
||||
/// Finds mesh data via hash, computes vertex outcodes against AABB from this+0x10,
|
||||
/// then iterates triangles: filters by visibility mask, trivial-rejects by outcode AND,
|
||||
/// adds survivors to global visible/render lists.
|
||||
pub fn performSpatialCulling(this: u32, key_data: u32, key_size: u32) callconv(.{ .x86_thiscall = .{} }) u32 {
|
||||
if (g_guard.* == 0) return 0;
|
||||
|
||||
const hash_table = g_guard.*;
|
||||
const hash_entry = @call(.never_tail, FindOrCreateHashEntry, .{
|
||||
hash_table, key_data, key_size,
|
||||
readU32(this + 4), readU32(this + 8), readU32(this + 0xC),
|
||||
});
|
||||
if (hash_entry == 0) return 0;
|
||||
|
||||
// Load AABB from *(this+0x10) -- 6 floats: minX, minY, minZ, maxX, maxY, maxZ
|
||||
const bounds_ptr = readU32(this + 0x10);
|
||||
const min_x = readF32(bounds_ptr);
|
||||
const min_y = readF32(bounds_ptr + 4);
|
||||
const min_z = readF32(bounds_ptr + 8);
|
||||
const max_x = readF32(bounds_ptr + 12);
|
||||
const max_y = readF32(bounds_ptr + 16);
|
||||
const max_z = readF32(bounds_ptr + 20);
|
||||
|
||||
// Phase 1: Compute per-vertex outcodes
|
||||
var cull_flags: [452]u8 = undefined;
|
||||
_ = computeAllOutcodes(hash_entry, min_x, max_x, min_y, max_y, min_z, max_z, &cull_flags);
|
||||
|
||||
// Phase 2: Iterate triangles
|
||||
const tri_count: u32 = @as(u32, readU16(hash_entry + 0x18A4));
|
||||
const filter_mask = readU16(this + 0x14);
|
||||
const visited_base = readU32(this + 4);
|
||||
|
||||
var ti: u32 = 0;
|
||||
while (ti < tri_count) : (ti += 1) {
|
||||
// Visibility mask filter
|
||||
const vis_flags = readU16(hash_entry + 0x1FAE + ti * 2);
|
||||
if ((vis_flags & filter_mask) != 0) continue;
|
||||
|
||||
// Per-vertex visited filter
|
||||
const tri_base_idx = readU16(hash_entry + 0x2206 + ti * 2);
|
||||
const visited_byte = @as(*u8, @ptrFromInt(visited_base + @as(u32, tri_base_idx) * 2));
|
||||
if ((visited_byte.* & @as(u8, @truncate(filter_mask))) != 0) continue;
|
||||
|
||||
// Check global visible list capacity
|
||||
if (g_visible_count.* >= 0x2000) {
|
||||
const flags_ptr = readU32(this);
|
||||
if (flags_ptr != 0) {
|
||||
const p: *u32 = @ptrFromInt(flags_ptr);
|
||||
p.* |= 1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Add to visible list
|
||||
g_visible_list[g_visible_count.*] = tri_base_idx;
|
||||
g_visible_count.* += 1;
|
||||
visited_byte.* |= 0x80;
|
||||
|
||||
// Frustum test: AND of 3 vertex outcodes. If any bit shared, fully outside.
|
||||
const idx0 = readU16(hash_entry + 0x18A6 + ti * 6);
|
||||
const idx1 = readU16(hash_entry + 0x18A8 + ti * 6);
|
||||
const idx2 = readU16(hash_entry + 0x18AA + ti * 6);
|
||||
// Note: decompiler shows idx offsets as 0x18A6, +0xC54*2, +0x18AA
|
||||
// which is 0x18A6 (idx0), 0x18A8 (idx1), 0x18AA (idx2) -- stride 6 = 3 u16 per tri
|
||||
|
||||
if ((cull_flags[idx0] & cull_flags[idx1] & cull_flags[idx2] & 0x3F) == 0) {
|
||||
g_render_list[g_render_count.*] = tri_base_idx;
|
||||
g_render_count.* += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// SSE vector helpers for Moller-Trumbore
|
||||
// =============================================================================
|
||||
|
||||
inline fn loadVec3(addr: u32) V4 {
|
||||
return .{ readF32(addr), readF32(addr + 4), readF32(addr + 8), 0 };
|
||||
}
|
||||
|
||||
inline fn cross(a: V4, b: V4) V4 {
|
||||
const Mask = @Vector(4, i32);
|
||||
const a_yzx: V4 = @shuffle(f32, a, undefined, Mask{ 1, 2, 0, 3 });
|
||||
const a_zxy: V4 = @shuffle(f32, a, undefined, Mask{ 2, 0, 1, 3 });
|
||||
const b_yzx: V4 = @shuffle(f32, b, undefined, Mask{ 1, 2, 0, 3 });
|
||||
const b_zxy: V4 = @shuffle(f32, b, undefined, Mask{ 2, 0, 1, 3 });
|
||||
return a_yzx * b_zxy - a_zxy * b_yzx;
|
||||
}
|
||||
|
||||
inline fn dot3(a: V4, b: V4) f32 {
|
||||
const p = a * b;
|
||||
return p[0] + p[1] + p[2];
|
||||
}
|
||||
|
||||
/// performCollisionDetection (0x6B88E0)
|
||||
/// __thiscall(this, keyData, keySize) -> u32. RET 0x8.
|
||||
///
|
||||
/// Fully inlined SSE rewrite. No external calls except FindOrCreateHashEntry.
|
||||
/// Moller-Trumbore ray-triangle intersection is inlined with SSE cross/dot,
|
||||
/// eliminating 4 SetVector3 calls and the ray_tri function call per triangle.
|
||||
pub fn performCollisionDetectionSSE(this: u32, key_data: u32, key_size: u32) callconv(.{ .x86_thiscall = .{} }) u32 {
|
||||
if (g_guard.* == 0) return 0;
|
||||
|
||||
const hash_table = g_guard.*;
|
||||
const hash_entry = @call(.never_tail, FindOrCreateHashEntry, .{
|
||||
hash_table, key_data, key_size,
|
||||
readU32(this + 4), readU32(this + 8), readU32(this + 0xC),
|
||||
});
|
||||
if (hash_entry == 0) return 0;
|
||||
|
||||
// Load and sort AABB extents from this+0x18..0x2C
|
||||
var ax0 = readF32(this + 0x18);
|
||||
var ax1 = readF32(this + 0x24);
|
||||
if (ax1 < ax0) {
|
||||
const tmp = ax0;
|
||||
ax0 = ax1;
|
||||
ax1 = tmp;
|
||||
}
|
||||
var ay0 = readF32(this + 0x1C);
|
||||
var ay1 = readF32(this + 0x28);
|
||||
if (ay1 < ay0) {
|
||||
const tmp = ay0;
|
||||
ay0 = ay1;
|
||||
ay1 = tmp;
|
||||
}
|
||||
var az0 = readF32(this + 0x20);
|
||||
var az1 = readF32(this + 0x2C);
|
||||
if (az1 < az0) {
|
||||
const tmp = az0;
|
||||
az0 = az1;
|
||||
az1 = tmp;
|
||||
}
|
||||
|
||||
// Phase 1: Compute per-vertex outcodes
|
||||
var cull_flags: [452]u8 = undefined;
|
||||
_ = computeAllOutcodes(hash_entry, ax0, ax1, ay0, ay1, az0, az1, &cull_flags);
|
||||
|
||||
// Phase 2: Iterate triangles with inline ray-tri test
|
||||
const tri_count: u32 = @as(u32, readU16(hash_entry + 0x18A4));
|
||||
const collision_mask = readU16(this + 0x50);
|
||||
const visited_base = readU32(this + 4);
|
||||
const vert_pool = hash_entry + 8;
|
||||
|
||||
// Ray: origin at this+0x00 (position), direction at this+0x0C (3 floats)
|
||||
// Original uses param_1 = ESI which points to a 6-float struct:
|
||||
// [0..2] = ray origin, [3..5] = ray direction
|
||||
// The call site passes this+0x30 as the ray struct
|
||||
const ray_origin = loadVec3(this + 0x30);
|
||||
const ray_dir = loadVec3(this + 0x3C);
|
||||
|
||||
// Epsilon for barycentric bounds: original uses +/- param_6 (0.002)
|
||||
const eps: f32 = 0.002;
|
||||
const neg_eps: f32 = -eps;
|
||||
const one_plus_eps: f32 = 1.0 + eps;
|
||||
|
||||
var ti: u32 = 0;
|
||||
while (ti < tri_count) : (ti += 1) {
|
||||
const vis_flags = readU16(hash_entry + 0x1FAE + ti * 2);
|
||||
if ((vis_flags & collision_mask) != 0) continue;
|
||||
|
||||
const tri_base_idx = readU16(hash_entry + 0x2206 + ti * 2);
|
||||
const visited_addr = visited_base + @as(u32, tri_base_idx) * 2;
|
||||
const visited_byte = @as(*u8, @ptrFromInt(visited_addr));
|
||||
if ((visited_byte.* & @as(u8, @truncate(collision_mask))) != 0) continue;
|
||||
|
||||
// Add to visible list and mark visited
|
||||
g_visible_list[g_visible_count.*] = tri_base_idx;
|
||||
g_visible_count.* += 1;
|
||||
visited_byte.* |= 0x80;
|
||||
|
||||
// Frustum outcode test
|
||||
const idx_base = hash_entry + 0x18A6 + ti * 6;
|
||||
const vi0: u32 = readU16(idx_base);
|
||||
const vi1: u32 = readU16(idx_base + 2);
|
||||
const vi2: u32 = readU16(idx_base + 4);
|
||||
|
||||
if ((cull_flags[vi0] & cull_flags[vi1] & cull_flags[vi2] & 0x3F) != 0) continue;
|
||||
|
||||
// =====================================================================
|
||||
// Inline Moller-Trumbore ray-triangle intersection (SSE)
|
||||
// Deferred divide: compare u_raw and v_raw against det-scaled bounds
|
||||
// to avoid the 1/det divide on the reject path.
|
||||
// =====================================================================
|
||||
|
||||
const v0 = loadVec3(vert_pool + vi0 * 12);
|
||||
const v1 = loadVec3(vert_pool + vi1 * 12);
|
||||
const v2 = loadVec3(vert_pool + vi2 * 12);
|
||||
|
||||
const edge1 = v1 - v0;
|
||||
const edge2 = v2 - v0;
|
||||
const pvec = cross(ray_dir, edge2);
|
||||
const det = dot3(edge1, pvec);
|
||||
|
||||
// Original thresholds: reject if det is in (-1e-6, 1e-6) dead zone
|
||||
if (det > -1e-6 and det < 1e-6) continue;
|
||||
|
||||
const inv_det = 1.0 / det;
|
||||
const tvec = ray_origin - v0;
|
||||
|
||||
const u = dot3(tvec, pvec) * inv_det;
|
||||
if (u < neg_eps or u > one_plus_eps) continue;
|
||||
|
||||
const qvec = cross(tvec, edge1);
|
||||
const v = dot3(ray_dir, qvec) * inv_det;
|
||||
if (v < neg_eps or (u + v) > one_plus_eps) continue;
|
||||
|
||||
const t = dot3(edge2, qvec) * inv_det;
|
||||
|
||||
if (t >= 0.0 and t < readF32(this + 0x4C)) {
|
||||
// Update closest hit
|
||||
@as(*align(1) f32, @ptrFromInt(this + 0x4C)).* = t;
|
||||
g_render_list[0] = tri_base_idx;
|
||||
g_render_count.* = 1;
|
||||
|
||||
// Write scaled distance, clamped to max
|
||||
const result_ptr: *align(1) f32 = @ptrFromInt(readU32(this + 0x10));
|
||||
const scaled = t * readF32(this + 0x48);
|
||||
const clamp = readF32(this + 0x14);
|
||||
result_ptr.* = if (scaled <= clamp) scaled else clamp;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
//! SSE-optimized entity update functions -- compiled ReleaseFast.
|
||||
//!
|
||||
//! Reimplements UpdateEntityAndChunksPositions (0x6AFAD0) and
|
||||
//! updateEntitiesInBounds (0x6C1F70).
|
||||
//!
|
||||
//! Main optimization: SSE dot product for view-depth computation,
|
||||
//! tighter control flow, reduced function call overhead.
|
||||
|
||||
const V4 = @Vector(4, f32);
|
||||
|
||||
fn readU32(addr: u32) u32 {
|
||||
return @as(*align(1) const u32, @ptrFromInt(addr)).*;
|
||||
}
|
||||
fn readI32(addr: u32) i32 {
|
||||
return @as(*align(1) const i32, @ptrFromInt(addr)).*;
|
||||
}
|
||||
fn readF32(addr: u32) f32 {
|
||||
return @as(*align(1) const f32, @ptrFromInt(addr)).*;
|
||||
}
|
||||
fn writeU32(addr: u32, val: u32) void {
|
||||
@as(*align(1) u32, @ptrFromInt(addr)).* = val;
|
||||
}
|
||||
fn writeF32(addr: u32, val: f32) void {
|
||||
@as(*align(1) f32, @ptrFromInt(addr)).* = val;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Game function declarations (resolved at link time via absolute address)
|
||||
// =========================================================================
|
||||
|
||||
// recycleVertexBuffer: fastcall(ECX=bufPtr, EDX=sizePtr), RET
|
||||
const recycleVertexBuffer = @as(*const fn (u32, u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x6AE9A0));
|
||||
// check_instances_active: fastcall(ECX=instanceMgr) -> ptr, RET
|
||||
const check_instances_active = @as(*const fn (u32) callconv(.{ .x86_fastcall = .{} }) u32, @ptrFromInt(0x6B2900));
|
||||
// store_all_instance_buffers: fastcall(ECX=instanceMgr), RET
|
||||
const store_all_instance_buffers = @as(*const fn (u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x6B28E0));
|
||||
// return_object_to_pool: fastcall(ECX=instanceMgr), RET
|
||||
const return_object_to_pool = @as(*const fn (u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x6B2030));
|
||||
// ReturnChunkBuffers: fastcall(ECX=bufPtr, EDX=sizePtr), RET
|
||||
const ReturnChunkBuffers = @as(*const fn (u32, u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x68CD50));
|
||||
// IsPointInsideBounds: fastcall(ECX=point, EDX=bounds) -> u32, RET
|
||||
const IsPointInsideBounds = @as(*const fn (u32, u32) callconv(.{ .x86_fastcall = .{} }) u32, @ptrFromInt(0x699330));
|
||||
// AddObjectToSpatialList: fastcall(ECX=entityPtr, EDX=posPtr), RET
|
||||
const AddObjectToSpatialList = @as(*const fn (u32, u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x6818B0));
|
||||
// AddToSpatialGrid: fastcall(ECX=objPtr), RET
|
||||
const AddToSpatialGrid = @as(*const fn (u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x6816F0));
|
||||
// CopyChunkBounds: thiscall(ECX=chunk, stack=outBounds), RET 0x4
|
||||
const CopyChunkBounds = @as(*const fn (u32, u32) callconv(.{ .x86_thiscall = .{} }) void, @ptrFromInt(0x68DF40));
|
||||
// AddToLayeredSpatialGrid: fastcall(ECX=chunk, EDX=idx, stack=posPtr), RET 0x4
|
||||
const AddToLayeredSpatialGrid = @as(*const fn (u32, u32, u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x681970));
|
||||
|
||||
// ComplexMemoryCleanupAndRelease: fastcall(ECX=memObj)
|
||||
const ComplexMemoryCleanupAndRelease = @as(*const fn (u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x6A0510));
|
||||
// destroySecondaryGameObject: fastcall(ECX=entityPtr)
|
||||
const destroySecondaryGameObject = @as(*const fn (u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x6A6A00));
|
||||
|
||||
// Game globals
|
||||
const g_viewCoeffs: u32 = 0xC7BCB0; // 4 floats: a, b, c, d for dot product
|
||||
const g_renderFlags: *const u8 = @ptrFromInt(0x867960 + 8); // actually at different offset
|
||||
const g_deltaTime: *const u32 = @ptrFromInt(0xC62510); // PTR_00c62510
|
||||
const g_timerThreshold: *const f32 = @ptrFromInt(0x80A1E8); // _DAT_0080a1e8
|
||||
|
||||
// =========================================================================
|
||||
// UpdateEntityAndChunksPositions (0x6AFAD0)
|
||||
// __fastcall(ECX=entityPtr), RET
|
||||
// =========================================================================
|
||||
export fn updateEntityAndChunksPositions(ent: u32) callconv(.{ .x86_fastcall = .{} }) void {
|
||||
// Dot product: depth = a*x + b*y + c*z + d - offset
|
||||
const depth = readF32(g_viewCoeffs) * readF32(ent + 0x5C) +
|
||||
readF32(g_viewCoeffs + 4) * readF32(ent + 0x60) +
|
||||
readF32(g_viewCoeffs + 8) * readF32(ent + 0x64) +
|
||||
readF32(g_viewCoeffs + 12) - readF32(ent + 0x68);
|
||||
writeF32(ent + 0x78, depth);
|
||||
|
||||
// Render distance flag
|
||||
writeU32(ent + 0xB8, readU32(0x867964));
|
||||
if ((@as(*const u8, @ptrFromInt(0xC7B2A4)).* & 4) != 0 and readF32(0x867960) < depth) {
|
||||
writeU32(ent + 0xB8, readU32(0x867968));
|
||||
}
|
||||
|
||||
// Timer accumulation
|
||||
const dt_bits = g_deltaTime.*;
|
||||
const dt: f32 = @bitCast(dt_bits);
|
||||
const timer = readF32(ent + 0xAC) + dt;
|
||||
writeF32(ent + 0xAC, timer);
|
||||
const threshold = g_timerThreshold.*;
|
||||
|
||||
// Vertex buffer recycling
|
||||
if (threshold < timer and readU32(ent + 0x14C) != 0) {
|
||||
@call(.never_tail, recycleVertexBuffer, .{ ent + 0x14C, ent + 0x150 });
|
||||
}
|
||||
|
||||
// Instance buffer management
|
||||
const inst_mgr = readU32(ent + 0xC0);
|
||||
if (inst_mgr != 0) {
|
||||
if (1.0 < readF32(ent + 0xAC)) {
|
||||
const active = @call(.never_tail, check_instances_active, .{inst_mgr});
|
||||
if (active != 0) {
|
||||
@call(.never_tail, store_all_instance_buffers, .{inst_mgr});
|
||||
}
|
||||
}
|
||||
if (threshold < readF32(ent + 0xAC)) {
|
||||
@call(.never_tail, return_object_to_pool, .{inst_mgr});
|
||||
writeU32(ent + 0xC0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Chunk timer loop (4 chunks at ent+0x118, stride 4)
|
||||
inline for (0..4) |ci| {
|
||||
const chunk = readU32(ent + 0x118 + ci * 4);
|
||||
if (chunk != 0) {
|
||||
const chunk_timer = readF32(chunk + 0x30) + dt;
|
||||
writeF32(chunk + 0x30, chunk_timer);
|
||||
if (readU32(chunk + 0x400) != 0 and threshold < chunk_timer) {
|
||||
@call(.never_tail, ReturnChunkBuffers, .{ chunk + 0x400, chunk + 0x404 });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bounds check and spatial grid registration
|
||||
const bx = readF32(ent + 0x44);
|
||||
const by = readF32(ent + 0x48);
|
||||
const bz = readF32(ent + 0x4C);
|
||||
if (bx <= readF32(0xC7CB68) and by <= readF32(0xC7CB6C) and bz <= readF32(0xC7CB70)) {
|
||||
const inside = @call(.never_tail, IsPointInsideBounds, .{ ent + 0x50, 0xC7CB5C });
|
||||
if (inside != 0) {
|
||||
// Compute position from animation data
|
||||
const anim_idx = readU32(0x86B580 + readU32(0xC7F294) * 4);
|
||||
const anim_base = ent + 0x83C + @as(u32, @bitCast(anim_idx)) * 0xC;
|
||||
var pos: [3]f32 = undefined;
|
||||
pos[0] = readF32(anim_base) + readF32(ent + 0x6C);
|
||||
pos[1] = readF32(anim_base + 4) + readF32(ent + 0x70);
|
||||
pos[2] = readF32(anim_base + 8) + readF32(ent + 0x74);
|
||||
|
||||
@call(.never_tail, AddObjectToSpatialList, .{ ent, @intFromPtr(&pos) });
|
||||
|
||||
// Walk sub-object linked list
|
||||
var node = readU32(ent + 0xE4);
|
||||
if ((node & 1) != 0 or node == 0) node = 0;
|
||||
while ((node & 1) == 0 and node != 0) {
|
||||
const obj = readU32(node + 4);
|
||||
if ((@as(*const u8, @ptrFromInt(obj + 0xC)).* & 0x80) != 0 and (readU32(obj + 0x88) != 0 or readU32(obj + 0x174) != 0)) {
|
||||
@call(.never_tail, AddToSpatialGrid, .{obj});
|
||||
}
|
||||
node = readU32(readU32(ent + 0xDC) + node + 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Chunk bounds + spatial grid loop (4 chunks)
|
||||
inline for (0..4) |ci| {
|
||||
const chunk = readU32(ent + 0x118 + ci * 4);
|
||||
if (chunk != 0) {
|
||||
var bounds: [6]f32 = undefined;
|
||||
@call(.never_tail, CopyChunkBounds, .{ chunk, @intFromPtr(&bounds) });
|
||||
|
||||
// Check if chunk bounds intersect the world region
|
||||
if (bounds[0] <= readF32(0xC7CB68) and bounds[1] <= readF32(0xC7CB6C) and
|
||||
bounds[2] <= readF32(0xC7CB70) and readF32(0xC7CB5C) <= bounds[3] and
|
||||
readF32(0xC7CB60) <= bounds[4] and readF32(0xC7CB64) <= bounds[5])
|
||||
{
|
||||
var center: [3]f32 = undefined;
|
||||
center[0] = (bounds[3] + bounds[0]) * 0.5;
|
||||
center[1] = (bounds[4] + bounds[1]) * 0.5;
|
||||
center[2] = (bounds[5] + bounds[2]) * 0.5;
|
||||
@call(.never_tail, AddToLayeredSpatialGrid, .{ chunk, @as(u32, ci), @intFromPtr(¢er) });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// updateEntitiesInBounds (0x6C1F70)
|
||||
// __thiscall(ECX=this, stack=param_1), RET 0x4
|
||||
// =========================================================================
|
||||
export fn updateEntitiesInBoundsSSE(this: u32, param_1: u32) callconv(.{ .x86_thiscall = .{} }) void {
|
||||
var node = readU32(this + 0x274);
|
||||
if ((node & 1) != 0 or node == 0) node = 0;
|
||||
|
||||
while ((node & 1) == 0 and node != 0) {
|
||||
const entity = readU32(node + 4);
|
||||
const next = readU32(readU32(this + 0x26C) + 4 + node);
|
||||
|
||||
// Prefetch next node's entity data while we process this one
|
||||
if ((next & 1) == 0 and next != 0) {
|
||||
const next_entity = readU32(next + 4);
|
||||
@prefetch(@as([*]const u8, @ptrFromInt(next_entity + 0x44)), .{ .locality = 1 });
|
||||
@prefetch(@as([*]const u8, @ptrFromInt(next_entity + 0x8C)), .{ .locality = 1 });
|
||||
}
|
||||
|
||||
// Bounds check: entity chunk coords vs global region
|
||||
const cx = readI32(entity + 0x8C);
|
||||
const cy = readI32(entity + 0x90);
|
||||
|
||||
if (cx < readI32(0xC63278) or readI32(0xC63280) < cx or
|
||||
cy < readI32(0xC63274) or readI32(0xC6327C) < cy)
|
||||
{
|
||||
// Out of bounds: destroy
|
||||
const slot_idx = readU32(entity + 0xB4); // entityPtr[0x2d] = +0xB4
|
||||
writeU32(this + slot_idx * 4 + 0x278, 0);
|
||||
@call(.never_tail, ComplexMemoryCleanupAndRelease, .{node});
|
||||
@call(.never_tail, destroySecondaryGameObject, .{entity});
|
||||
} else {
|
||||
if (param_1 != 0) {
|
||||
// Call through game address so the Detour hook fires (enables A/B timing)
|
||||
const entPosHooked = @as(*const fn (u32) callconv(.{ .x86_fastcall = .{} }) void, @ptrFromInt(0x6AFAD0));
|
||||
@call(.never_tail, entPosHooked, .{entity});
|
||||
}
|
||||
}
|
||||
node = next;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
//! filecache -- MPQ archive pointer cache for File_FindInArchive (0x6549a0)
|
||||
//!
|
||||
//! 2-way set-associative cache: hash(filename) picks a set of 2 entries.
|
||||
//! Filename stored and compared for collision safety. On eviction, the
|
||||
//! least recently used way is replaced.
|
||||
//!
|
||||
//! Lifecycle managed by the performance module (no own hooks or mutex).
|
||||
|
||||
const std = @import("std");
|
||||
const hook = @import("zhook");
|
||||
const logging = @import("../logging.zig");
|
||||
|
||||
pub inline fn rdtsc() u64 {
|
||||
var lo: u32 = undefined;
|
||||
var hi: u32 = undefined;
|
||||
asm volatile ("rdtsc"
|
||||
: [lo] "={eax}" (lo),
|
||||
[hi] "={edx}" (hi),
|
||||
);
|
||||
return @as(u64, hi) << 32 | lo;
|
||||
}
|
||||
|
||||
// FNV-1a 32-bit, raw bytes (no normalization). Different casings get different slots.
|
||||
// Finalizer mixes high bits into low bits for better slot distribution.
|
||||
pub fn hashPath(path: [*:0]const u8) u32 {
|
||||
var h: u32 = 0x811c9dc5;
|
||||
var i: usize = 0;
|
||||
while (path[i] != 0) : (i += 1) {
|
||||
h ^= path[i];
|
||||
h *%= 0x01000193;
|
||||
}
|
||||
h ^= h >> 16;
|
||||
return h;
|
||||
}
|
||||
|
||||
// Longest filename across all MPQ archives is 122 chars. 128 covers all.
|
||||
const CACHE_NAME_LEN = 128;
|
||||
const NUM_SETS = 32768; // power of 2
|
||||
const WAYS = 2; // 2-way set-associative
|
||||
|
||||
pub const ArchiveCacheEntry = struct {
|
||||
outer_archive: u32 = 0,
|
||||
inner_archive: u32 = 0,
|
||||
block_index: u32 = 0, // index into archive's block table, not raw pointer
|
||||
is_negative: bool = false,
|
||||
name: [CACHE_NAME_LEN]u8 = .{0} ** CACHE_NAME_LEN,
|
||||
name_len: u8 = 0,
|
||||
};
|
||||
|
||||
const CacheSet = struct {
|
||||
entries: [WAYS]ArchiveCacheEntry = .{ArchiveCacheEntry{}} ** WAYS,
|
||||
lru: u8 = 0,
|
||||
};
|
||||
|
||||
const WINAPI = @import("std").builtin.CallingConvention.winapi;
|
||||
extern "kernel32" fn GetProcessHeap() callconv(WINAPI) ?*anyopaque;
|
||||
extern "kernel32" fn HeapAlloc(hHeap: ?*anyopaque, dwFlags: u32, dwBytes: usize) callconv(WINAPI) ?[*]u8;
|
||||
extern "kernel32" fn HeapFree(hHeap: ?*anyopaque, dwFlags: u32, lpMem: *anyopaque) callconv(WINAPI) i32;
|
||||
|
||||
var cache: ?[*]CacheSet = null;
|
||||
var cache_entries: u32 = 0;
|
||||
var cache_hits: u64 = 0;
|
||||
var cache_negative_hits: u64 = 0;
|
||||
var cache_misses: u64 = 0;
|
||||
var cache_stale: u64 = 0;
|
||||
var hit_cycles: u64 = 0;
|
||||
var miss_cycles: u64 = 0;
|
||||
var cache_miss_p1: u64 = 0;
|
||||
var cache_miss_p2: u64 = 0;
|
||||
var cache_miss_p2_archive: u64 = 0;
|
||||
|
||||
/// Lookup: hash picks set, check both ways for name match.
|
||||
pub fn archiveCacheLookup(h: u32, path: [*:0]const u8) ?ArchiveCacheEntry {
|
||||
const c = cache orelse return null;
|
||||
const set_idx = h & (NUM_SETS - 1);
|
||||
const set = &c[set_idx];
|
||||
const span = std.mem.span(path);
|
||||
|
||||
for (0..WAYS) |w| {
|
||||
const entry = &set.entries[w];
|
||||
if (entry.name_len == 0) continue;
|
||||
if (span.len != entry.name_len) continue;
|
||||
const len: usize = entry.name_len;
|
||||
if (std.mem.eql(u8, entry.name[0..len], span[0..len])) {
|
||||
set.lru = @intCast(w ^ 1);
|
||||
return entry.*;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Recompute block_entry pointer from archive's current block table + cached index.
|
||||
/// block_entry = archive->block_table_data + index * 0x2C
|
||||
pub fn computeBlockEntry(archive: u32, index: u32) u32 {
|
||||
if (archive == 0) return 0;
|
||||
const block_table_base = hook.readMem(u32, archive + 0x290);
|
||||
if (block_table_base == 0) return 0;
|
||||
return block_table_base + index * 0x2C;
|
||||
}
|
||||
|
||||
pub fn archiveCacheInsert(h: u32, path: [*:0]const u8, outer: u32, inner: u32, block: u32, negative: bool) void {
|
||||
const c = cache orelse return;
|
||||
const span = std.mem.span(path);
|
||||
if (span.len > CACHE_NAME_LEN) return;
|
||||
|
||||
const set_idx = h & (NUM_SETS - 1);
|
||||
const set = &c[set_idx];
|
||||
|
||||
var target: u8 = set.lru;
|
||||
for (0..WAYS) |w| {
|
||||
if (set.entries[w].name_len == 0) {
|
||||
target = @intCast(w);
|
||||
cache_entries += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const entry = &set.entries[target];
|
||||
entry.outer_archive = outer;
|
||||
entry.inner_archive = inner;
|
||||
if (block != 0 and inner != 0) {
|
||||
const base = hook.readMem(u32, inner + 0x290);
|
||||
entry.block_index = if (base != 0) (block - base) / 0x2C else 0;
|
||||
} else {
|
||||
entry.block_index = 0;
|
||||
}
|
||||
entry.is_negative = negative;
|
||||
const len: u8 = @intCast(span.len);
|
||||
@memcpy(entry.name[0..len], span[0..len]);
|
||||
entry.name_len = len;
|
||||
set.lru = target ^ 1;
|
||||
}
|
||||
|
||||
pub fn recordCacheHit() void { cache_hits +|= 1; }
|
||||
pub fn addHitCycles(c: u64) void { hit_cycles +|= c; }
|
||||
pub fn addMissCycles(c: u64) void { miss_cycles +|= c; }
|
||||
pub fn recordNegativeHit() void { cache_negative_hits +|= 1; }
|
||||
pub fn recordCacheMiss() void { cache_misses +|= 1; }
|
||||
pub fn recordCacheStale() void { cache_stale +|= 1; }
|
||||
pub fn recordMissP1() void { cache_miss_p1 +|= 1; }
|
||||
pub fn recordMissP2() void { cache_miss_p2 +|= 1; }
|
||||
pub fn recordMissP2Archive() void { cache_miss_p2_archive +|= 1; }
|
||||
|
||||
pub fn getSlotOccupant(h: u32) ?[]const u8 {
|
||||
const c = cache orelse return null;
|
||||
const set_idx = h & (NUM_SETS - 1);
|
||||
const set = &c[set_idx];
|
||||
for (0..WAYS) |w| {
|
||||
if (set.entries[w].name_len != 0)
|
||||
return set.entries[w].name[0..set.entries[w].name_len];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// File_FindInArchive hook (0x6549a0)
|
||||
// =============================================================================
|
||||
|
||||
const FileFindFn = fn (u32, u32, u32, u32, u32, u32, u32) callconv(hook.cc.fastcall) u32;
|
||||
var file_find_hook: hook.Detour(FileFindFn) = .{};
|
||||
|
||||
fn fileFindDetour(
|
||||
archive_or_group: u32,
|
||||
filename_ptr: u32,
|
||||
flags: u32,
|
||||
out_inner_archive: u32,
|
||||
out_outer_archive: u32,
|
||||
out_block_entry: u32,
|
||||
out_disk_path: u32,
|
||||
) callconv(hook.cc.fastcall) u32 {
|
||||
if (filename_ptr == 0)
|
||||
return file_find_hook.callOriginal(.{ archive_or_group, filename_ptr, flags, out_inner_archive, out_outer_archive, out_block_entry, out_disk_path });
|
||||
|
||||
const tsc_start = rdtsc();
|
||||
const path: [*:0]const u8 = @ptrFromInt(filename_ptr);
|
||||
const h = hashPath(path);
|
||||
|
||||
cache_check: {
|
||||
const cached = archiveCacheLookup(h, path) orelse break :cache_check;
|
||||
|
||||
if (cached.is_negative and archive_or_group == 0) {
|
||||
recordNegativeHit();
|
||||
if (out_outer_archive != 0) @as(*align(1) u32, @ptrFromInt(out_outer_archive)).* = 0;
|
||||
if (out_inner_archive != 0) @as(*align(1) u32, @ptrFromInt(out_inner_archive)).* = 0;
|
||||
if (out_block_entry != 0) @as(*align(1) u32, @ptrFromInt(out_block_entry)).* = 0;
|
||||
hook.call(fn (u32) callconv(hook.cc.stdcall) void, 0x64e850, .{2});
|
||||
addHitCycles(rdtsc() - tsc_start);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!cached.is_negative) {
|
||||
if (archive_or_group == 0 and out_outer_archive != 0) {
|
||||
const valid_outer = if (cached.outer_archive != 0)
|
||||
hook.call(fn (u32, u32) callconv(hook.cc.fastcall) u32, 0x650780, .{ cached.outer_archive, 0 })
|
||||
else
|
||||
0;
|
||||
if (valid_outer == 0 and cached.outer_archive != 0) break :cache_check;
|
||||
@as(*align(1) u32, @ptrFromInt(out_outer_archive)).* = valid_outer;
|
||||
if (out_inner_archive != 0 and cached.inner_archive != 0) {
|
||||
const valid_inner = hook.call(fn (u32, u32) callconv(hook.cc.fastcall) u32, 0x650780, .{ cached.inner_archive, 0 });
|
||||
@as(*align(1) u32, @ptrFromInt(out_inner_archive)).* = valid_inner;
|
||||
} else if (out_inner_archive != 0) {
|
||||
@as(*align(1) u32, @ptrFromInt(out_inner_archive)).* = 0;
|
||||
}
|
||||
if (out_block_entry != 0) {
|
||||
@as(*align(1) u32, @ptrFromInt(out_block_entry)).* = computeBlockEntry(cached.inner_archive, cached.block_index);
|
||||
}
|
||||
recordCacheHit();
|
||||
addHitCycles(rdtsc() - tsc_start);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (archive_or_group != 0 and (archive_or_group == cached.outer_archive or archive_or_group == cached.inner_archive)) {
|
||||
if (out_outer_archive != 0 and cached.outer_archive != 0) {
|
||||
const valid = hook.call(fn (u32, u32) callconv(hook.cc.fastcall) u32, 0x650780, .{ cached.outer_archive, 0 });
|
||||
if (valid == 0) break :cache_check;
|
||||
@as(*align(1) u32, @ptrFromInt(out_outer_archive)).* = valid;
|
||||
} else if (out_outer_archive != 0) {
|
||||
@as(*align(1) u32, @ptrFromInt(out_outer_archive)).* = 0;
|
||||
}
|
||||
if (out_inner_archive != 0 and cached.inner_archive != 0) {
|
||||
const valid = hook.call(fn (u32, u32) callconv(hook.cc.fastcall) u32, 0x650780, .{ cached.inner_archive, 0 });
|
||||
if (valid == 0) break :cache_check;
|
||||
@as(*align(1) u32, @ptrFromInt(out_inner_archive)).* = valid;
|
||||
} else if (out_inner_archive != 0) {
|
||||
@as(*align(1) u32, @ptrFromInt(out_inner_archive)).* = 0;
|
||||
}
|
||||
if (out_block_entry != 0) {
|
||||
@as(*align(1) u32, @ptrFromInt(out_block_entry)).* = computeBlockEntry(cached.inner_archive, cached.block_index);
|
||||
}
|
||||
recordCacheHit();
|
||||
addHitCycles(rdtsc() - tsc_start);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
recordCacheMiss();
|
||||
const ret = file_find_hook.callOriginal(.{ archive_or_group, filename_ptr, flags, out_inner_archive, out_outer_archive, out_block_entry, out_disk_path });
|
||||
addMissCycles(rdtsc() - tsc_start);
|
||||
|
||||
if (archive_or_group != 0 and out_inner_archive != 0 and out_block_entry != 0) {
|
||||
if (ret == 1) {
|
||||
const inner = hook.readMem(u32, out_inner_archive);
|
||||
const block = hook.readMem(u32, out_block_entry);
|
||||
archiveCacheInsert(h, path, archive_or_group, inner, block, false);
|
||||
}
|
||||
}
|
||||
if (archive_or_group == 0 and ret == 0) {
|
||||
archiveCacheInsert(h, path, 0, 0, 0, true);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
pub fn install() bool {
|
||||
const size = NUM_SETS * @sizeOf(CacheSet);
|
||||
const heap = GetProcessHeap() orelse return false;
|
||||
const ptr = HeapAlloc(heap, 0x00000008, size) orelse return false; // HEAP_ZERO_MEMORY
|
||||
cache = @alignCast(@ptrCast(ptr));
|
||||
return file_find_hook.attach(0x6549a0, &fileFindDetour) == .ok;
|
||||
}
|
||||
|
||||
pub fn remove() void {
|
||||
file_find_hook.detach();
|
||||
if (cache) |c| {
|
||||
if (GetProcessHeap()) |heap| _ = HeapFree(heap, 0, @ptrCast(c));
|
||||
cache = null;
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Stats
|
||||
// =============================================================================
|
||||
|
||||
fn pct(part: u64, total: u64) u64 {
|
||||
if (total == 0) return 0;
|
||||
return part *| 1000 / total;
|
||||
}
|
||||
|
||||
fn resetStats() void {
|
||||
cache_hits = 0;
|
||||
cache_negative_hits = 0;
|
||||
cache_misses = 0;
|
||||
cache_stale = 0;
|
||||
cache_miss_p1 = 0;
|
||||
cache_miss_p2 = 0;
|
||||
cache_miss_p2_archive = 0;
|
||||
hit_cycles = 0;
|
||||
miss_cycles = 0;
|
||||
}
|
||||
|
||||
pub fn dumpStats(lg: *logging.Logger) void {
|
||||
const total = cache_hits + cache_negative_hits + cache_misses + cache_stale;
|
||||
if (total == 0) return;
|
||||
const hit_pct = pct(cache_hits + cache_negative_hits, total);
|
||||
const total_hit_calls = cache_hits + cache_negative_hits;
|
||||
const avg_hit = if (total_hit_calls > 0) hit_cycles / total_hit_calls else 0;
|
||||
const avg_miss = if (cache_misses > 0) miss_cycles / cache_misses else 0;
|
||||
const saved_us = if (avg_miss > avg_hit) total_hit_calls * (avg_miss - avg_hit) / 3000 else 0;
|
||||
if (saved_us >= 2000) {
|
||||
lg.fmt("file_cache: {d}.{d}% hit ({d}hit/{d}neg/{d}miss) {d} entries | saved {d}ms\n", .{
|
||||
hit_pct / 10, hit_pct % 10,
|
||||
cache_hits, cache_negative_hits, cache_misses, cache_entries,
|
||||
saved_us / 1000,
|
||||
});
|
||||
} else {
|
||||
lg.fmt("file_cache: {d}.{d}% hit ({d}hit/{d}neg/{d}miss) {d} entries | saved {d}us\n", .{
|
||||
hit_pct / 10, hit_pct % 10,
|
||||
cache_hits, cache_negative_hits, cache_misses, cache_entries,
|
||||
saved_us,
|
||||
});
|
||||
}
|
||||
resetStats();
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
//! inflate_hook — libdeflate replacement for WoW's zlib inflate.
|
||||
//!
|
||||
//! Hooks DecompressData_WithOptions (0x661A80). For type 0x02 (zlib) streams,
|
||||
//! uses libdeflate (~2.2x faster than stock zlib). Falls back to original on failure.
|
||||
//!
|
||||
//! Thread-safety: the decompressor struct contains mutable decode tables rebuilt
|
||||
//! per block, so each thread gets its own cached decompressor via Zig's native
|
||||
//! threadlocal (OS-managed TLS). This avoids both the race condition
|
||||
//! (shared decompressor -> wild writes) and the per-call alloc overhead.
|
||||
//!
|
||||
//! Benchmark results (84k calls, heavy load):
|
||||
//! stock=2681ms | per-call-alloc=1304ms (2.05x) | tls-cached=1194ms (2.2x)
|
||||
|
||||
const hook_lib = @import("zhook");
|
||||
const logging = @import("../logging.zig");
|
||||
|
||||
// libdeflate C API
|
||||
extern fn libdeflate_alloc_decompressor() ?*anyopaque;
|
||||
extern fn libdeflate_free_decompressor(?*anyopaque) void;
|
||||
extern fn libdeflate_zlib_decompress(?*anyopaque, [*]const u8, usize, [*]u8, usize, *usize) c_int;
|
||||
|
||||
var lib_available: bool = false;
|
||||
var log: logging.Logger = .{};
|
||||
|
||||
// --- Thread-local decompressor ---
|
||||
threadlocal var tls_decompressor: ?*anyopaque = null;
|
||||
|
||||
fn getTlsDecompressor() ?*anyopaque {
|
||||
if (tls_decompressor) |d| return d;
|
||||
const d = libdeflate_alloc_decompressor() orelse return null;
|
||||
tls_decompressor = d;
|
||||
return d;
|
||||
}
|
||||
|
||||
// --- Timing ---
|
||||
var fast_total_cycles: u64 = 0;
|
||||
var orig_total_cycles: u64 = 0;
|
||||
var call_count: u64 = 0;
|
||||
var success_count: u64 = 0;
|
||||
var fallback_count: u64 = 0;
|
||||
|
||||
inline fn rdtsc() u64 {
|
||||
var lo: u32 = undefined;
|
||||
var hi: u32 = undefined;
|
||||
asm volatile ("rdtsc" : [lo] "={eax}" (lo), [hi] "={edx}" (hi));
|
||||
return (@as(u64, hi) << 32) | lo;
|
||||
}
|
||||
|
||||
const DecompressFn = fn (u32, u32, u32, u32, u32) callconv(.{ .x86_stdcall = .{} }) u32;
|
||||
pub var decompress_hook: hook_lib.Detour(DecompressFn) = .{};
|
||||
|
||||
pub fn decompressDetour(out_buf: u32, out_size_ptr: u32, in_buf: u32, in_size: u32, flags: u32) callconv(.{ .x86_stdcall = .{} }) u32 {
|
||||
const in_ptr: [*]const u8 = @ptrFromInt(in_buf);
|
||||
const out_capacity = @as(*const u32, @ptrFromInt(out_size_ptr)).*;
|
||||
|
||||
// Accept type 0x02 with valid zlib CMF (deflate method = low nibble 0x08)
|
||||
if (lib_available and in_size > 3 and out_capacity > 0 and
|
||||
in_ptr[0] == 0x02 and (in_ptr[1] & 0x0F) == 0x08)
|
||||
{
|
||||
if (getTlsDecompressor()) |decomp| {
|
||||
var ld_out: usize = 0;
|
||||
const t1 = rdtsc();
|
||||
const ld_ret = libdeflate_zlib_decompress(
|
||||
decomp,
|
||||
in_ptr + 1,
|
||||
in_size - 1,
|
||||
@ptrFromInt(out_buf),
|
||||
out_capacity,
|
||||
&ld_out,
|
||||
);
|
||||
fast_total_cycles +|= rdtsc() - t1;
|
||||
|
||||
if (ld_ret == 0) {
|
||||
call_count +|= 1;
|
||||
success_count +|= 1;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to original
|
||||
call_count +|= 1;
|
||||
const t0 = rdtsc();
|
||||
const ret = decompress_hook.callOriginal(.{ out_buf, out_size_ptr, in_buf, in_size, flags });
|
||||
orig_total_cycles +|= rdtsc() - t0;
|
||||
fallback_count +|= 1;
|
||||
return ret;
|
||||
}
|
||||
|
||||
pub fn dumpStats() void {
|
||||
if (call_count == 0) return;
|
||||
const MS = 3_000_000;
|
||||
log.fmt("inflate: {d} calls | fast={d} ({d}ms) fallback={d} ({d}ms)\n", .{
|
||||
call_count, success_count, fast_total_cycles / MS, fallback_count, orig_total_cycles / MS,
|
||||
});
|
||||
fast_total_cycles = 0;
|
||||
orig_total_cycles = 0;
|
||||
call_count = 0;
|
||||
success_count = 0;
|
||||
fallback_count = 0;
|
||||
}
|
||||
|
||||
pub fn install() bool {
|
||||
const test_decomp = libdeflate_alloc_decompressor();
|
||||
if (test_decomp == null) return false;
|
||||
libdeflate_free_decompressor(test_decomp);
|
||||
lib_available = true;
|
||||
return decompress_hook.attach(0x661A80, &decompressDetour) == .ok;
|
||||
}
|
||||
|
||||
pub fn remove() void {
|
||||
decompress_hook.detach();
|
||||
lib_available = false;
|
||||
}
|
||||
@@ -0,0 +1,749 @@
|
||||
/*
|
||||
* common_defs.h
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef COMMON_DEFS_H
|
||||
#define COMMON_DEFS_H
|
||||
|
||||
#include "libdeflate.h"
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h> /* for size_t */
|
||||
#include <stdint.h>
|
||||
#ifdef _MSC_VER
|
||||
# include <intrin.h> /* for _BitScan*() and other intrinsics */
|
||||
# include <stdlib.h> /* for _byteswap_*() */
|
||||
/* Disable MSVC warnings that are expected. */
|
||||
/* /W2 */
|
||||
# pragma warning(disable : 4146) /* unary minus on unsigned type */
|
||||
/* /W3 */
|
||||
# pragma warning(disable : 4018) /* signed/unsigned mismatch */
|
||||
# pragma warning(disable : 4244) /* possible loss of data */
|
||||
# pragma warning(disable : 4267) /* possible loss of precision */
|
||||
# pragma warning(disable : 4310) /* cast truncates constant value */
|
||||
/* /W4 */
|
||||
# pragma warning(disable : 4100) /* unreferenced formal parameter */
|
||||
# pragma warning(disable : 4127) /* conditional expression is constant */
|
||||
# pragma warning(disable : 4189) /* local variable initialized but not referenced */
|
||||
# pragma warning(disable : 4232) /* nonstandard extension used */
|
||||
# pragma warning(disable : 4245) /* conversion from 'int' to 'unsigned int' */
|
||||
# pragma warning(disable : 4295) /* array too small to include terminating null */
|
||||
#endif
|
||||
#ifndef FREESTANDING
|
||||
# include <string.h> /* for memcpy() */
|
||||
#endif
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Target architecture */
|
||||
/* ========================================================================== */
|
||||
|
||||
/* If possible, define a compiler-independent ARCH_* macro. */
|
||||
#undef ARCH_X86_64
|
||||
#undef ARCH_X86_32
|
||||
#undef ARCH_ARM64
|
||||
#undef ARCH_ARM32
|
||||
#undef ARCH_RISCV
|
||||
#ifdef _MSC_VER
|
||||
/* Way too many things are broken in ARM64EC to pretend that it is x86_64. */
|
||||
# if defined(_M_X64) && !defined(_M_ARM64EC)
|
||||
# define ARCH_X86_64
|
||||
# elif defined(_M_IX86)
|
||||
# define ARCH_X86_32
|
||||
# elif defined(_M_ARM64)
|
||||
# define ARCH_ARM64
|
||||
# elif defined(_M_ARM)
|
||||
# define ARCH_ARM32
|
||||
# endif
|
||||
#else
|
||||
# if defined(__x86_64__)
|
||||
# define ARCH_X86_64
|
||||
# elif defined(__i386__)
|
||||
# define ARCH_X86_32
|
||||
# elif defined(__aarch64__)
|
||||
# define ARCH_ARM64
|
||||
# elif defined(__arm__)
|
||||
# define ARCH_ARM32
|
||||
# elif defined(__riscv)
|
||||
# define ARCH_RISCV
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Type definitions */
|
||||
/* ========================================================================== */
|
||||
|
||||
/* Fixed-width integer types */
|
||||
typedef uint8_t u8;
|
||||
typedef uint16_t u16;
|
||||
typedef uint32_t u32;
|
||||
typedef uint64_t u64;
|
||||
typedef int8_t s8;
|
||||
typedef int16_t s16;
|
||||
typedef int32_t s32;
|
||||
typedef int64_t s64;
|
||||
|
||||
/* ssize_t, if not available in <sys/types.h> */
|
||||
#ifdef _MSC_VER
|
||||
# ifdef _WIN64
|
||||
typedef long long ssize_t;
|
||||
# else
|
||||
typedef long ssize_t;
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Word type of the target architecture. Use 'size_t' instead of
|
||||
* 'unsigned long' to account for platforms such as Windows that use 32-bit
|
||||
* 'unsigned long' on 64-bit architectures.
|
||||
*/
|
||||
typedef size_t machine_word_t;
|
||||
|
||||
/* Number of bytes in a word */
|
||||
#define WORDBYTES ((int)sizeof(machine_word_t))
|
||||
|
||||
/* Number of bits in a word */
|
||||
#define WORDBITS (8 * WORDBYTES)
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Optional compiler features */
|
||||
/* ========================================================================== */
|
||||
|
||||
/* Compiler version checks. Only use when absolutely necessary. */
|
||||
#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
|
||||
# define GCC_PREREQ(major, minor) \
|
||||
(__GNUC__ > (major) || \
|
||||
(__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
|
||||
# if !GCC_PREREQ(4, 9)
|
||||
# error "gcc versions older than 4.9 are no longer supported"
|
||||
# endif
|
||||
#else
|
||||
# define GCC_PREREQ(major, minor) 0
|
||||
#endif
|
||||
#ifdef __clang__
|
||||
# ifdef __apple_build_version__
|
||||
# define CLANG_PREREQ(major, minor, apple_version) \
|
||||
(__apple_build_version__ >= (apple_version))
|
||||
# else
|
||||
# define CLANG_PREREQ(major, minor, apple_version) \
|
||||
(__clang_major__ > (major) || \
|
||||
(__clang_major__ == (major) && __clang_minor__ >= (minor)))
|
||||
# endif
|
||||
# if !CLANG_PREREQ(3, 9, 8000000)
|
||||
# error "clang versions older than 3.9 are no longer supported"
|
||||
# endif
|
||||
#else
|
||||
# define CLANG_PREREQ(major, minor, apple_version) 0
|
||||
#endif
|
||||
#ifdef _MSC_VER
|
||||
# define MSVC_PREREQ(version) (_MSC_VER >= (version))
|
||||
# if !MSVC_PREREQ(1900)
|
||||
# error "MSVC versions older than Visual Studio 2015 are no longer supported"
|
||||
# endif
|
||||
#else
|
||||
# define MSVC_PREREQ(version) 0
|
||||
#endif
|
||||
|
||||
/*
|
||||
* __has_attribute(attribute) - check whether the compiler supports the given
|
||||
* attribute (and also supports doing the check in the first place). Mostly
|
||||
* useful just for clang, since gcc didn't add this macro until gcc 5.
|
||||
*/
|
||||
#ifndef __has_attribute
|
||||
# define __has_attribute(attribute) 0
|
||||
#endif
|
||||
|
||||
/*
|
||||
* __has_builtin(builtin) - check whether the compiler supports the given
|
||||
* builtin (and also supports doing the check in the first place). Mostly
|
||||
* useful just for clang, since gcc didn't add this macro until gcc 10.
|
||||
*/
|
||||
#ifndef __has_builtin
|
||||
# define __has_builtin(builtin) 0
|
||||
#endif
|
||||
|
||||
/* inline - suggest that a function be inlined */
|
||||
#ifdef _MSC_VER
|
||||
# define inline __inline
|
||||
#endif /* else assume 'inline' is usable as-is */
|
||||
|
||||
/* forceinline - force a function to be inlined, if possible */
|
||||
#if defined(__GNUC__) || __has_attribute(always_inline)
|
||||
# define forceinline inline __attribute__((always_inline))
|
||||
#elif defined(_MSC_VER)
|
||||
# define forceinline __forceinline
|
||||
#else
|
||||
# define forceinline inline
|
||||
#endif
|
||||
|
||||
/* MAYBE_UNUSED - mark a function or variable as maybe unused */
|
||||
#if defined(__GNUC__) || __has_attribute(unused)
|
||||
# define MAYBE_UNUSED __attribute__((unused))
|
||||
#else
|
||||
# define MAYBE_UNUSED
|
||||
#endif
|
||||
|
||||
/* NORETURN - mark a function as never returning, e.g. due to calling abort() */
|
||||
#if defined(__GNUC__) || __has_attribute(noreturn)
|
||||
# define NORETURN __attribute__((noreturn))
|
||||
#else
|
||||
# define NORETURN
|
||||
#endif
|
||||
|
||||
/*
|
||||
* restrict - hint that writes only occur through the given pointer.
|
||||
*
|
||||
* Don't use MSVC's __restrict, since it has nonstandard behavior.
|
||||
* Standard restrict is okay, if it is supported.
|
||||
*/
|
||||
#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 201112L)
|
||||
# if defined(__GNUC__) || defined(__clang__)
|
||||
# define restrict __restrict__
|
||||
# else
|
||||
# define restrict
|
||||
# endif
|
||||
#endif /* else assume 'restrict' is usable as-is */
|
||||
|
||||
/* likely(expr) - hint that an expression is usually true */
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_expect)
|
||||
# define likely(expr) __builtin_expect(!!(expr), 1)
|
||||
#else
|
||||
# define likely(expr) (expr)
|
||||
#endif
|
||||
|
||||
/* unlikely(expr) - hint that an expression is usually false */
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_expect)
|
||||
# define unlikely(expr) __builtin_expect(!!(expr), 0)
|
||||
#else
|
||||
# define unlikely(expr) (expr)
|
||||
#endif
|
||||
|
||||
/* prefetchr(addr) - prefetch into L1 cache for read */
|
||||
#undef prefetchr
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
|
||||
# define prefetchr(addr) __builtin_prefetch((addr), 0)
|
||||
#elif defined(_MSC_VER)
|
||||
# if defined(ARCH_X86_32) || defined(ARCH_X86_64)
|
||||
# define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
|
||||
# elif defined(ARCH_ARM64)
|
||||
# define prefetchr(addr) __prefetch2((addr), 0x00 /* prfop=PLDL1KEEP */)
|
||||
# elif defined(ARCH_ARM32)
|
||||
# define prefetchr(addr) __prefetch(addr)
|
||||
# endif
|
||||
#endif
|
||||
#ifndef prefetchr
|
||||
# define prefetchr(addr)
|
||||
#endif
|
||||
|
||||
/* prefetchw(addr) - prefetch into L1 cache for write */
|
||||
#undef prefetchw
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
|
||||
# define prefetchw(addr) __builtin_prefetch((addr), 1)
|
||||
#elif defined(_MSC_VER)
|
||||
# if defined(ARCH_X86_32) || defined(ARCH_X86_64)
|
||||
# define prefetchw(addr) _m_prefetchw(addr)
|
||||
# elif defined(ARCH_ARM64)
|
||||
# define prefetchw(addr) __prefetch2((addr), 0x10 /* prfop=PSTL1KEEP */)
|
||||
# elif defined(ARCH_ARM32)
|
||||
# define prefetchw(addr) __prefetchw(addr)
|
||||
# endif
|
||||
#endif
|
||||
#ifndef prefetchw
|
||||
# define prefetchw(addr)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* _aligned_attribute(n) - declare that the annotated variable, or variables of
|
||||
* the annotated type, must be aligned on n-byte boundaries.
|
||||
*/
|
||||
#undef _aligned_attribute
|
||||
#if defined(__GNUC__) || __has_attribute(aligned)
|
||||
# define _aligned_attribute(n) __attribute__((aligned(n)))
|
||||
#elif defined(_MSC_VER)
|
||||
# define _aligned_attribute(n) __declspec(align(n))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* _target_attribute(attrs) - override the compilation target for a function.
|
||||
*
|
||||
* This accepts one or more comma-separated suffixes to the -m prefix jointly
|
||||
* forming the name of a machine-dependent option. On gcc-like compilers, this
|
||||
* enables codegen for the given targets, including arbitrary compiler-generated
|
||||
* code as well as the corresponding intrinsics. On other compilers this macro
|
||||
* expands to nothing, though MSVC allows intrinsics to be used anywhere anyway.
|
||||
*/
|
||||
#if defined(__GNUC__) || __has_attribute(target)
|
||||
# define _target_attribute(attrs) __attribute__((target(attrs)))
|
||||
#else
|
||||
# define _target_attribute(attrs)
|
||||
#endif
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Miscellaneous macros */
|
||||
/* ========================================================================== */
|
||||
|
||||
#define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
|
||||
#define MIN(a, b) ((a) <= (b) ? (a) : (b))
|
||||
#define MAX(a, b) ((a) >= (b) ? (a) : (b))
|
||||
#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
|
||||
#define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
|
||||
#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
|
||||
#define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Endianness handling */
|
||||
/* ========================================================================== */
|
||||
|
||||
/*
|
||||
* CPU_IS_LITTLE_ENDIAN() - 1 if the CPU is little endian, or 0 if it is big
|
||||
* endian. When possible this is a compile-time macro that can be used in
|
||||
* preprocessor conditionals. As a fallback, a generic method is used that
|
||||
* can't be used in preprocessor conditionals but should still be optimized out.
|
||||
*/
|
||||
#if defined(__BYTE_ORDER__) /* gcc v4.6+ and clang */
|
||||
# define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
#elif defined(_MSC_VER)
|
||||
# define CPU_IS_LITTLE_ENDIAN() true
|
||||
#else
|
||||
static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
|
||||
{
|
||||
union {
|
||||
u32 w;
|
||||
u8 b;
|
||||
} u;
|
||||
|
||||
u.w = 1;
|
||||
return u.b;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* bswap16(v) - swap the bytes of a 16-bit integer */
|
||||
static forceinline u16 bswap16(u16 v)
|
||||
{
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
|
||||
return __builtin_bswap16(v);
|
||||
#elif defined(_MSC_VER)
|
||||
return _byteswap_ushort(v);
|
||||
#else
|
||||
return (v << 8) | (v >> 8);
|
||||
#endif
|
||||
}
|
||||
|
||||
/* bswap32(v) - swap the bytes of a 32-bit integer */
|
||||
static forceinline u32 bswap32(u32 v)
|
||||
{
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
|
||||
return __builtin_bswap32(v);
|
||||
#elif defined(_MSC_VER)
|
||||
return _byteswap_ulong(v);
|
||||
#else
|
||||
return ((v & 0x000000FF) << 24) |
|
||||
((v & 0x0000FF00) << 8) |
|
||||
((v & 0x00FF0000) >> 8) |
|
||||
((v & 0xFF000000) >> 24);
|
||||
#endif
|
||||
}
|
||||
|
||||
/* bswap64(v) - swap the bytes of a 64-bit integer */
|
||||
static forceinline u64 bswap64(u64 v)
|
||||
{
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
|
||||
return __builtin_bswap64(v);
|
||||
#elif defined(_MSC_VER)
|
||||
return _byteswap_uint64(v);
|
||||
#else
|
||||
return ((v & 0x00000000000000FF) << 56) |
|
||||
((v & 0x000000000000FF00) << 40) |
|
||||
((v & 0x0000000000FF0000) << 24) |
|
||||
((v & 0x00000000FF000000) << 8) |
|
||||
((v & 0x000000FF00000000) >> 8) |
|
||||
((v & 0x0000FF0000000000) >> 24) |
|
||||
((v & 0x00FF000000000000) >> 40) |
|
||||
((v & 0xFF00000000000000) >> 56);
|
||||
#endif
|
||||
}
|
||||
|
||||
#define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
|
||||
#define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
|
||||
#define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
|
||||
#define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
|
||||
#define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
|
||||
#define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Unaligned memory accesses */
|
||||
/* ========================================================================== */
|
||||
|
||||
/*
|
||||
* UNALIGNED_ACCESS_IS_FAST() - 1 if unaligned memory accesses can be performed
|
||||
* efficiently on the target platform, otherwise 0.
|
||||
*/
|
||||
#if (defined(__GNUC__) || defined(__clang__)) && \
|
||||
(defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
|
||||
defined(__ARM_FEATURE_UNALIGNED) || \
|
||||
defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
|
||||
defined(__riscv_misaligned_fast) || \
|
||||
/*
|
||||
* For all compilation purposes, WebAssembly behaves like any other CPU
|
||||
* instruction set. Even though WebAssembly engine might be running on
|
||||
* top of different actual CPU architectures, the WebAssembly spec
|
||||
* itself permits unaligned access and it will be fast on most of those
|
||||
* platforms, and simulated at the engine level on others, so it's
|
||||
* worth treating it as a CPU architecture with fast unaligned access.
|
||||
*/ defined(__wasm__))
|
||||
# define UNALIGNED_ACCESS_IS_FAST 1
|
||||
#elif defined(_MSC_VER)
|
||||
# define UNALIGNED_ACCESS_IS_FAST 1
|
||||
#else
|
||||
# define UNALIGNED_ACCESS_IS_FAST 0
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Implementing unaligned memory accesses using memcpy() is portable, and it
|
||||
* usually gets optimized appropriately by modern compilers. I.e., each
|
||||
* memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled to a load or store
|
||||
* instruction, not to an actual function call.
|
||||
*
|
||||
* We no longer use the "packed struct" approach to unaligned accesses, as that
|
||||
* is nonstandard, has unclear semantics, and doesn't receive enough testing
|
||||
* (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
|
||||
*
|
||||
* arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception
|
||||
* where memcpy() generates inefficient code
|
||||
* (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366). However, we no longer
|
||||
* consider that one case important enough to maintain different code for.
|
||||
* If you run into it, please just use a newer version of gcc (or use clang).
|
||||
*/
|
||||
|
||||
#ifdef FREESTANDING
|
||||
# define MEMCOPY __builtin_memcpy
|
||||
#else
|
||||
# define MEMCOPY memcpy
|
||||
#endif
|
||||
|
||||
/* Unaligned loads and stores without endianness conversion */
|
||||
|
||||
#define DEFINE_UNALIGNED_TYPE(type) \
|
||||
static forceinline type \
|
||||
load_##type##_unaligned(const void *p) \
|
||||
{ \
|
||||
type v; \
|
||||
\
|
||||
MEMCOPY(&v, p, sizeof(v)); \
|
||||
return v; \
|
||||
} \
|
||||
\
|
||||
static forceinline void \
|
||||
store_##type##_unaligned(type v, void *p) \
|
||||
{ \
|
||||
MEMCOPY(p, &v, sizeof(v)); \
|
||||
}
|
||||
|
||||
DEFINE_UNALIGNED_TYPE(u16)
|
||||
DEFINE_UNALIGNED_TYPE(u32)
|
||||
DEFINE_UNALIGNED_TYPE(u64)
|
||||
DEFINE_UNALIGNED_TYPE(machine_word_t)
|
||||
|
||||
#undef MEMCOPY
|
||||
|
||||
#define load_word_unaligned load_machine_word_t_unaligned
|
||||
#define store_word_unaligned store_machine_word_t_unaligned
|
||||
|
||||
/* Unaligned loads with endianness conversion */
|
||||
|
||||
static forceinline u16
|
||||
get_unaligned_le16(const u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST)
|
||||
return le16_bswap(load_u16_unaligned(p));
|
||||
else
|
||||
return ((u16)p[1] << 8) | p[0];
|
||||
}
|
||||
|
||||
static forceinline u16
|
||||
get_unaligned_be16(const u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST)
|
||||
return be16_bswap(load_u16_unaligned(p));
|
||||
else
|
||||
return ((u16)p[0] << 8) | p[1];
|
||||
}
|
||||
|
||||
static forceinline u32
|
||||
get_unaligned_le32(const u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST)
|
||||
return le32_bswap(load_u32_unaligned(p));
|
||||
else
|
||||
return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
|
||||
((u32)p[1] << 8) | p[0];
|
||||
}
|
||||
|
||||
static forceinline u32
|
||||
get_unaligned_be32(const u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST)
|
||||
return be32_bswap(load_u32_unaligned(p));
|
||||
else
|
||||
return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
|
||||
((u32)p[2] << 8) | p[3];
|
||||
}
|
||||
|
||||
static forceinline u64
|
||||
get_unaligned_le64(const u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST)
|
||||
return le64_bswap(load_u64_unaligned(p));
|
||||
else
|
||||
return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
|
||||
((u64)p[5] << 40) | ((u64)p[4] << 32) |
|
||||
((u64)p[3] << 24) | ((u64)p[2] << 16) |
|
||||
((u64)p[1] << 8) | p[0];
|
||||
}
|
||||
|
||||
static forceinline machine_word_t
|
||||
get_unaligned_leword(const u8 *p)
|
||||
{
|
||||
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
|
||||
if (WORDBITS == 32)
|
||||
return get_unaligned_le32(p);
|
||||
else
|
||||
return get_unaligned_le64(p);
|
||||
}
|
||||
|
||||
/* Unaligned stores with endianness conversion */
|
||||
|
||||
static forceinline void
|
||||
put_unaligned_le16(u16 v, u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST) {
|
||||
store_u16_unaligned(le16_bswap(v), p);
|
||||
} else {
|
||||
p[0] = (u8)(v >> 0);
|
||||
p[1] = (u8)(v >> 8);
|
||||
}
|
||||
}
|
||||
|
||||
static forceinline void
|
||||
put_unaligned_be16(u16 v, u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST) {
|
||||
store_u16_unaligned(be16_bswap(v), p);
|
||||
} else {
|
||||
p[0] = (u8)(v >> 8);
|
||||
p[1] = (u8)(v >> 0);
|
||||
}
|
||||
}
|
||||
|
||||
static forceinline void
|
||||
put_unaligned_le32(u32 v, u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST) {
|
||||
store_u32_unaligned(le32_bswap(v), p);
|
||||
} else {
|
||||
p[0] = (u8)(v >> 0);
|
||||
p[1] = (u8)(v >> 8);
|
||||
p[2] = (u8)(v >> 16);
|
||||
p[3] = (u8)(v >> 24);
|
||||
}
|
||||
}
|
||||
|
||||
static forceinline void
|
||||
put_unaligned_be32(u32 v, u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST) {
|
||||
store_u32_unaligned(be32_bswap(v), p);
|
||||
} else {
|
||||
p[0] = (u8)(v >> 24);
|
||||
p[1] = (u8)(v >> 16);
|
||||
p[2] = (u8)(v >> 8);
|
||||
p[3] = (u8)(v >> 0);
|
||||
}
|
||||
}
|
||||
|
||||
static forceinline void
|
||||
put_unaligned_le64(u64 v, u8 *p)
|
||||
{
|
||||
if (UNALIGNED_ACCESS_IS_FAST) {
|
||||
store_u64_unaligned(le64_bswap(v), p);
|
||||
} else {
|
||||
p[0] = (u8)(v >> 0);
|
||||
p[1] = (u8)(v >> 8);
|
||||
p[2] = (u8)(v >> 16);
|
||||
p[3] = (u8)(v >> 24);
|
||||
p[4] = (u8)(v >> 32);
|
||||
p[5] = (u8)(v >> 40);
|
||||
p[6] = (u8)(v >> 48);
|
||||
p[7] = (u8)(v >> 56);
|
||||
}
|
||||
}
|
||||
|
||||
static forceinline void
|
||||
put_unaligned_leword(machine_word_t v, u8 *p)
|
||||
{
|
||||
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
|
||||
if (WORDBITS == 32)
|
||||
put_unaligned_le32(v, p);
|
||||
else
|
||||
put_unaligned_le64(v, p);
|
||||
}
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Bit manipulation functions */
|
||||
/* ========================================================================== */
|
||||
|
||||
/*
|
||||
* Bit Scan Reverse (BSR) - find the 0-based index (relative to the least
|
||||
* significant end) of the *most* significant 1 bit in the input value. The
|
||||
* input value must be nonzero!
|
||||
*/
|
||||
|
||||
static forceinline unsigned
|
||||
bsr32(u32 v)
|
||||
{
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_clz)
|
||||
return 31 - __builtin_clz(v);
|
||||
#elif defined(_MSC_VER)
|
||||
unsigned long i;
|
||||
|
||||
_BitScanReverse(&i, v);
|
||||
return i;
|
||||
#else
|
||||
unsigned i = 0;
|
||||
|
||||
while ((v >>= 1) != 0)
|
||||
i++;
|
||||
return i;
|
||||
#endif
|
||||
}
|
||||
|
||||
static forceinline unsigned
|
||||
bsr64(u64 v)
|
||||
{
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_clzll)
|
||||
return 63 - __builtin_clzll(v);
|
||||
#elif defined(_MSC_VER) && defined(_WIN64)
|
||||
unsigned long i;
|
||||
|
||||
_BitScanReverse64(&i, v);
|
||||
return i;
|
||||
#else
|
||||
unsigned i = 0;
|
||||
|
||||
while ((v >>= 1) != 0)
|
||||
i++;
|
||||
return i;
|
||||
#endif
|
||||
}
|
||||
|
||||
static forceinline unsigned
|
||||
bsrw(machine_word_t v)
|
||||
{
|
||||
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
|
||||
if (WORDBITS == 32)
|
||||
return bsr32(v);
|
||||
else
|
||||
return bsr64(v);
|
||||
}
|
||||
|
||||
/*
|
||||
* Bit Scan Forward (BSF) - find the 0-based index (relative to the least
|
||||
* significant end) of the *least* significant 1 bit in the input value. The
|
||||
* input value must be nonzero!
|
||||
*/
|
||||
|
||||
static forceinline unsigned
|
||||
bsf32(u32 v)
|
||||
{
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_ctz)
|
||||
return __builtin_ctz(v);
|
||||
#elif defined(_MSC_VER)
|
||||
unsigned long i;
|
||||
|
||||
_BitScanForward(&i, v);
|
||||
return i;
|
||||
#else
|
||||
unsigned i = 0;
|
||||
|
||||
for (; (v & 1) == 0; v >>= 1)
|
||||
i++;
|
||||
return i;
|
||||
#endif
|
||||
}
|
||||
|
||||
static forceinline unsigned
|
||||
bsf64(u64 v)
|
||||
{
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
|
||||
return __builtin_ctzll(v);
|
||||
#elif defined(_MSC_VER) && defined(_WIN64)
|
||||
unsigned long i;
|
||||
|
||||
_BitScanForward64(&i, v);
|
||||
return i;
|
||||
#else
|
||||
unsigned i = 0;
|
||||
|
||||
for (; (v & 1) == 0; v >>= 1)
|
||||
i++;
|
||||
return i;
|
||||
#endif
|
||||
}
|
||||
|
||||
static forceinline unsigned
|
||||
bsfw(machine_word_t v)
|
||||
{
|
||||
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
|
||||
if (WORDBITS == 32)
|
||||
return bsf32(v);
|
||||
else
|
||||
return bsf64(v);
|
||||
}
|
||||
|
||||
/*
|
||||
* rbit32(v): reverse the bits in a 32-bit integer. This doesn't have a
|
||||
* fallback implementation; use '#ifdef rbit32' to check if this is available.
|
||||
*/
|
||||
#undef rbit32
|
||||
#if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
|
||||
(__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
|
||||
static forceinline u32
|
||||
rbit32(u32 v)
|
||||
{
|
||||
__asm__("rbit %0, %1" : "=r" (v) : "r" (v));
|
||||
return v;
|
||||
}
|
||||
#define rbit32 rbit32
|
||||
#elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
|
||||
static forceinline u32
|
||||
rbit32(u32 v)
|
||||
{
|
||||
__asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
|
||||
return v;
|
||||
}
|
||||
#define rbit32 rbit32
|
||||
#endif
|
||||
|
||||
#endif /* COMMON_DEFS_H */
|
||||
@@ -0,0 +1,162 @@
|
||||
/*
|
||||
* adler32.c - Adler-32 checksum algorithm
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "lib_common.h"
|
||||
|
||||
/* The Adler-32 divisor, or "base", value */
|
||||
#define DIVISOR 65521
|
||||
|
||||
/*
|
||||
* MAX_CHUNK_LEN is the most bytes that can be processed without the possibility
|
||||
* of s2 overflowing when it is represented as an unsigned 32-bit integer. This
|
||||
* value was computed using the following Python script:
|
||||
*
|
||||
* divisor = 65521
|
||||
* count = 0
|
||||
* s1 = divisor - 1
|
||||
* s2 = divisor - 1
|
||||
* while True:
|
||||
* s1 += 0xFF
|
||||
* s2 += s1
|
||||
* if s2 > 0xFFFFFFFF:
|
||||
* break
|
||||
* count += 1
|
||||
* print(count)
|
||||
*
|
||||
* Note that to get the correct worst-case value, we must assume that every byte
|
||||
* has value 0xFF and that s1 and s2 started with the highest possible values
|
||||
* modulo the divisor.
|
||||
*/
|
||||
#define MAX_CHUNK_LEN 5552
|
||||
|
||||
/*
|
||||
* Update the Adler-32 values s1 and s2 using n bytes from p, update p to p + n,
|
||||
* update n to 0, and reduce s1 and s2 mod DIVISOR. It is assumed that neither
|
||||
* s1 nor s2 can overflow before the reduction at the end, i.e. n plus any bytes
|
||||
* already processed after the last reduction must not exceed MAX_CHUNK_LEN.
|
||||
*
|
||||
* This uses only portable C code. This is used as a fallback when a vectorized
|
||||
* implementation of Adler-32 (e.g. AVX2) is unavailable on the platform.
|
||||
*
|
||||
* Some of the vectorized implementations also use this to handle the end of the
|
||||
* data when the data isn't evenly divisible by the length the vectorized code
|
||||
* works on. To avoid compiler errors about target-specific option mismatches
|
||||
* when this is used in that way, this is a macro rather than a function.
|
||||
*
|
||||
* Although this is unvectorized, this does include an optimization where the
|
||||
* main loop processes four bytes at a time using a strategy similar to that
|
||||
* used by vectorized implementations. This provides increased instruction-
|
||||
* level parallelism compared to the traditional 's1 += *p++; s2 += s1;'.
|
||||
*/
|
||||
#define ADLER32_CHUNK(s1, s2, p, n) \
|
||||
do { \
|
||||
if (n >= 4) { \
|
||||
u32 s1_sum = 0; \
|
||||
u32 byte_0_sum = 0; \
|
||||
u32 byte_1_sum = 0; \
|
||||
u32 byte_2_sum = 0; \
|
||||
u32 byte_3_sum = 0; \
|
||||
\
|
||||
do { \
|
||||
s1_sum += s1; \
|
||||
s1 += p[0] + p[1] + p[2] + p[3]; \
|
||||
byte_0_sum += p[0]; \
|
||||
byte_1_sum += p[1]; \
|
||||
byte_2_sum += p[2]; \
|
||||
byte_3_sum += p[3]; \
|
||||
p += 4; \
|
||||
n -= 4; \
|
||||
} while (n >= 4); \
|
||||
s2 += (4 * (s1_sum + byte_0_sum)) + (3 * byte_1_sum) + \
|
||||
(2 * byte_2_sum) + byte_3_sum; \
|
||||
} \
|
||||
for (; n; n--, p++) { \
|
||||
s1 += *p; \
|
||||
s2 += s1; \
|
||||
} \
|
||||
s1 %= DIVISOR; \
|
||||
s2 %= DIVISOR; \
|
||||
} while (0)
|
||||
|
||||
static u32 MAYBE_UNUSED
|
||||
adler32_generic(u32 adler, const u8 *p, size_t len)
|
||||
{
|
||||
u32 s1 = adler & 0xFFFF;
|
||||
u32 s2 = adler >> 16;
|
||||
|
||||
while (len) {
|
||||
size_t n = MIN(len, MAX_CHUNK_LEN & ~3);
|
||||
|
||||
len -= n;
|
||||
ADLER32_CHUNK(s1, s2, p, n);
|
||||
}
|
||||
|
||||
return (s2 << 16) | s1;
|
||||
}
|
||||
|
||||
/* Include architecture-specific implementation(s) if available. */
|
||||
#undef DEFAULT_IMPL
|
||||
#undef arch_select_adler32_func
|
||||
typedef u32 (*adler32_func_t)(u32 adler, const u8 *p, size_t len);
|
||||
#if defined(ARCH_ARM32) || defined(ARCH_ARM64)
|
||||
# include "arm/adler32_impl.h"
|
||||
#elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
|
||||
# include "x86/adler32_impl.h"
|
||||
#endif
|
||||
|
||||
#ifndef DEFAULT_IMPL
|
||||
# define DEFAULT_IMPL adler32_generic
|
||||
#endif
|
||||
|
||||
#ifdef arch_select_adler32_func
|
||||
static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len);
|
||||
|
||||
static volatile adler32_func_t adler32_impl = dispatch_adler32;
|
||||
|
||||
/* Choose the best implementation at runtime. */
|
||||
static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len)
|
||||
{
|
||||
adler32_func_t f = arch_select_adler32_func();
|
||||
|
||||
if (f == NULL)
|
||||
f = DEFAULT_IMPL;
|
||||
|
||||
adler32_impl = f;
|
||||
return f(adler, p, len);
|
||||
}
|
||||
#else
|
||||
/* The best implementation is statically known, so call it directly. */
|
||||
#define adler32_impl DEFAULT_IMPL
|
||||
#endif
|
||||
|
||||
LIBDEFLATEAPI u32
|
||||
libdeflate_adler32(u32 adler, const void *buffer, size_t len)
|
||||
{
|
||||
if (buffer == NULL) /* Return initial value. */
|
||||
return 1;
|
||||
return adler32_impl(adler, buffer, len);
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
* cpu_features_common.h - code shared by all lib/$arch/cpu_features.c
|
||||
*
|
||||
* Copyright 2020 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef LIB_CPU_FEATURES_COMMON_H
|
||||
#define LIB_CPU_FEATURES_COMMON_H
|
||||
|
||||
#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
|
||||
/* for strdup() and strtok_r() */
|
||||
# undef _ANSI_SOURCE
|
||||
# ifndef __APPLE__
|
||||
# undef _GNU_SOURCE
|
||||
# define _GNU_SOURCE
|
||||
# endif
|
||||
# include <stdio.h>
|
||||
# include <stdlib.h>
|
||||
# include <string.h>
|
||||
#endif
|
||||
|
||||
#include "lib_common.h"
|
||||
|
||||
struct cpu_feature {
|
||||
u32 bit;
|
||||
const char *name;
|
||||
};
|
||||
|
||||
#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
|
||||
/* Disable any features that are listed in $LIBDEFLATE_DISABLE_CPU_FEATURES. */
|
||||
static inline void
|
||||
disable_cpu_features_for_testing(u32 *features,
|
||||
const struct cpu_feature *feature_table,
|
||||
size_t feature_table_length)
|
||||
{
|
||||
char *env_value, *strbuf, *p, *saveptr = NULL;
|
||||
size_t i;
|
||||
|
||||
env_value = getenv("LIBDEFLATE_DISABLE_CPU_FEATURES");
|
||||
if (!env_value)
|
||||
return;
|
||||
strbuf = strdup(env_value);
|
||||
if (!strbuf)
|
||||
abort();
|
||||
p = strtok_r(strbuf, ",", &saveptr);
|
||||
while (p) {
|
||||
for (i = 0; i < feature_table_length; i++) {
|
||||
if (strcmp(p, feature_table[i].name) == 0) {
|
||||
*features &= ~feature_table[i].bit;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (i == feature_table_length) {
|
||||
fprintf(stderr,
|
||||
"unrecognized feature in LIBDEFLATE_DISABLE_CPU_FEATURES: \"%s\"\n",
|
||||
p);
|
||||
abort();
|
||||
}
|
||||
p = strtok_r(NULL, ",", &saveptr);
|
||||
}
|
||||
free(strbuf);
|
||||
}
|
||||
#else /* TEST_SUPPORT__DO_NOT_USE */
|
||||
static inline void
|
||||
disable_cpu_features_for_testing(u32 *features,
|
||||
const struct cpu_feature *feature_table,
|
||||
size_t feature_table_length)
|
||||
{
|
||||
}
|
||||
#endif /* !TEST_SUPPORT__DO_NOT_USE */
|
||||
|
||||
#endif /* LIB_CPU_FEATURES_COMMON_H */
|
||||
@@ -0,0 +1,777 @@
|
||||
/*
|
||||
* decompress_template.h
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This is the actual DEFLATE decompression routine, lifted out of
|
||||
* deflate_decompress.c so that it can be compiled multiple times with different
|
||||
* target instruction sets.
|
||||
*/
|
||||
|
||||
#ifndef ATTRIBUTES
|
||||
# define ATTRIBUTES
|
||||
#endif
|
||||
#ifndef EXTRACT_VARBITS
|
||||
# define EXTRACT_VARBITS(word, count) ((word) & BITMASK(count))
|
||||
#endif
|
||||
#ifndef EXTRACT_VARBITS8
|
||||
# define EXTRACT_VARBITS8(word, count) ((word) & BITMASK((u8)(count)))
|
||||
#endif
|
||||
|
||||
static ATTRIBUTES MAYBE_UNUSED enum libdeflate_result
|
||||
FUNCNAME(struct libdeflate_decompressor * restrict d,
|
||||
const void * restrict in, size_t in_nbytes,
|
||||
void * restrict out, size_t out_nbytes_avail,
|
||||
size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
|
||||
{
|
||||
u8 *out_next = out;
|
||||
u8 * const out_end = out_next + out_nbytes_avail;
|
||||
u8 * const out_fastloop_end =
|
||||
out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN);
|
||||
|
||||
/* Input bitstream state; see deflate_decompress.c for documentation */
|
||||
const u8 *in_next = in;
|
||||
const u8 * const in_end = in_next + in_nbytes;
|
||||
const u8 * const in_fastloop_end =
|
||||
in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ);
|
||||
bitbuf_t bitbuf = 0;
|
||||
bitbuf_t saved_bitbuf;
|
||||
u32 bitsleft = 0;
|
||||
size_t overread_count = 0;
|
||||
|
||||
bool is_final_block;
|
||||
unsigned block_type;
|
||||
unsigned num_litlen_syms;
|
||||
unsigned num_offset_syms;
|
||||
bitbuf_t litlen_tablemask;
|
||||
u32 entry;
|
||||
|
||||
next_block:
|
||||
/* Starting to read the next block */
|
||||
;
|
||||
|
||||
STATIC_ASSERT(CAN_CONSUME(1 + 2 + 5 + 5 + 4 + 3));
|
||||
REFILL_BITS();
|
||||
|
||||
/* BFINAL: 1 bit */
|
||||
is_final_block = bitbuf & BITMASK(1);
|
||||
|
||||
/* BTYPE: 2 bits */
|
||||
block_type = (bitbuf >> 1) & BITMASK(2);
|
||||
|
||||
if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) {
|
||||
|
||||
/* Dynamic Huffman block */
|
||||
|
||||
/* The order in which precode lengths are stored */
|
||||
static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = {
|
||||
16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
|
||||
};
|
||||
|
||||
unsigned num_explicit_precode_lens;
|
||||
unsigned i;
|
||||
|
||||
/* Read the codeword length counts. */
|
||||
|
||||
STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 257 + BITMASK(5));
|
||||
num_litlen_syms = 257 + ((bitbuf >> 3) & BITMASK(5));
|
||||
|
||||
STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 1 + BITMASK(5));
|
||||
num_offset_syms = 1 + ((bitbuf >> 8) & BITMASK(5));
|
||||
|
||||
STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == 4 + BITMASK(4));
|
||||
num_explicit_precode_lens = 4 + ((bitbuf >> 13) & BITMASK(4));
|
||||
|
||||
d->static_codes_loaded = false;
|
||||
|
||||
/*
|
||||
* Read the precode codeword lengths.
|
||||
*
|
||||
* A 64-bit bitbuffer is just one bit too small to hold the
|
||||
* maximum number of precode lens, so to minimize branches we
|
||||
* merge one len with the previous fields.
|
||||
*/
|
||||
STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);
|
||||
if (CAN_CONSUME(3 * (DEFLATE_NUM_PRECODE_SYMS - 1))) {
|
||||
d->u.precode_lens[deflate_precode_lens_permutation[0]] =
|
||||
(bitbuf >> 17) & BITMASK(3);
|
||||
bitbuf >>= 20;
|
||||
bitsleft -= 20;
|
||||
REFILL_BITS();
|
||||
i = 1;
|
||||
do {
|
||||
d->u.precode_lens[deflate_precode_lens_permutation[i]] =
|
||||
bitbuf & BITMASK(3);
|
||||
bitbuf >>= 3;
|
||||
bitsleft -= 3;
|
||||
} while (++i < num_explicit_precode_lens);
|
||||
} else {
|
||||
bitbuf >>= 17;
|
||||
bitsleft -= 17;
|
||||
i = 0;
|
||||
do {
|
||||
if ((u8)bitsleft < 3)
|
||||
REFILL_BITS();
|
||||
d->u.precode_lens[deflate_precode_lens_permutation[i]] =
|
||||
bitbuf & BITMASK(3);
|
||||
bitbuf >>= 3;
|
||||
bitsleft -= 3;
|
||||
} while (++i < num_explicit_precode_lens);
|
||||
}
|
||||
for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)
|
||||
d->u.precode_lens[deflate_precode_lens_permutation[i]] = 0;
|
||||
|
||||
/* Build the decode table for the precode. */
|
||||
SAFETY_CHECK(build_precode_decode_table(d));
|
||||
|
||||
/* Decode the litlen and offset codeword lengths. */
|
||||
i = 0;
|
||||
do {
|
||||
unsigned presym;
|
||||
u8 rep_val;
|
||||
unsigned rep_count;
|
||||
|
||||
if ((u8)bitsleft < DEFLATE_MAX_PRE_CODEWORD_LEN + 7)
|
||||
REFILL_BITS();
|
||||
|
||||
/*
|
||||
* The code below assumes that the precode decode table
|
||||
* doesn't have any subtables.
|
||||
*/
|
||||
STATIC_ASSERT(PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN);
|
||||
|
||||
/* Decode the next precode symbol. */
|
||||
entry = d->u.l.precode_decode_table[
|
||||
bitbuf & BITMASK(DEFLATE_MAX_PRE_CODEWORD_LEN)];
|
||||
bitbuf >>= (u8)entry;
|
||||
bitsleft -= entry; /* optimization: subtract full entry */
|
||||
presym = entry >> 16;
|
||||
|
||||
if (presym < 16) {
|
||||
/* Explicit codeword length */
|
||||
d->u.l.lens[i++] = presym;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Run-length encoded codeword lengths */
|
||||
|
||||
/*
|
||||
* Note: we don't need to immediately verify that the
|
||||
* repeat count doesn't overflow the number of elements,
|
||||
* since we've sized the lens array to have enough extra
|
||||
* space to allow for the worst-case overrun (138 zeroes
|
||||
* when only 1 length was remaining).
|
||||
*
|
||||
* In the case of the small repeat counts (presyms 16
|
||||
* and 17), it is fastest to always write the maximum
|
||||
* number of entries. That gets rid of branches that
|
||||
* would otherwise be required.
|
||||
*
|
||||
* It is not just because of the numerical order that
|
||||
* our checks go in the order 'presym < 16', 'presym ==
|
||||
* 16', and 'presym == 17'. For typical data this is
|
||||
* ordered from most frequent to least frequent case.
|
||||
*/
|
||||
STATIC_ASSERT(DEFLATE_MAX_LENS_OVERRUN == 138 - 1);
|
||||
|
||||
if (presym == 16) {
|
||||
/* Repeat the previous length 3 - 6 times. */
|
||||
SAFETY_CHECK(i != 0);
|
||||
rep_val = d->u.l.lens[i - 1];
|
||||
STATIC_ASSERT(3 + BITMASK(2) == 6);
|
||||
rep_count = 3 + (bitbuf & BITMASK(2));
|
||||
bitbuf >>= 2;
|
||||
bitsleft -= 2;
|
||||
d->u.l.lens[i + 0] = rep_val;
|
||||
d->u.l.lens[i + 1] = rep_val;
|
||||
d->u.l.lens[i + 2] = rep_val;
|
||||
d->u.l.lens[i + 3] = rep_val;
|
||||
d->u.l.lens[i + 4] = rep_val;
|
||||
d->u.l.lens[i + 5] = rep_val;
|
||||
i += rep_count;
|
||||
} else if (presym == 17) {
|
||||
/* Repeat zero 3 - 10 times. */
|
||||
STATIC_ASSERT(3 + BITMASK(3) == 10);
|
||||
rep_count = 3 + (bitbuf & BITMASK(3));
|
||||
bitbuf >>= 3;
|
||||
bitsleft -= 3;
|
||||
d->u.l.lens[i + 0] = 0;
|
||||
d->u.l.lens[i + 1] = 0;
|
||||
d->u.l.lens[i + 2] = 0;
|
||||
d->u.l.lens[i + 3] = 0;
|
||||
d->u.l.lens[i + 4] = 0;
|
||||
d->u.l.lens[i + 5] = 0;
|
||||
d->u.l.lens[i + 6] = 0;
|
||||
d->u.l.lens[i + 7] = 0;
|
||||
d->u.l.lens[i + 8] = 0;
|
||||
d->u.l.lens[i + 9] = 0;
|
||||
i += rep_count;
|
||||
} else {
|
||||
/* Repeat zero 11 - 138 times. */
|
||||
STATIC_ASSERT(11 + BITMASK(7) == 138);
|
||||
rep_count = 11 + (bitbuf & BITMASK(7));
|
||||
bitbuf >>= 7;
|
||||
bitsleft -= 7;
|
||||
memset(&d->u.l.lens[i], 0,
|
||||
rep_count * sizeof(d->u.l.lens[i]));
|
||||
i += rep_count;
|
||||
}
|
||||
} while (i < num_litlen_syms + num_offset_syms);
|
||||
|
||||
/* Unnecessary, but check this for consistency with zlib. */
|
||||
SAFETY_CHECK(i == num_litlen_syms + num_offset_syms);
|
||||
|
||||
} else if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) {
|
||||
u16 len, nlen;
|
||||
|
||||
/*
|
||||
* Uncompressed block: copy 'len' bytes literally from the input
|
||||
* buffer to the output buffer.
|
||||
*/
|
||||
|
||||
bitsleft -= 3; /* for BTYPE and BFINAL */
|
||||
|
||||
/*
|
||||
* Align the bitstream to the next byte boundary. This means
|
||||
* the next byte boundary as if we were reading a byte at a
|
||||
* time. Therefore, we have to rewind 'in_next' by any bytes
|
||||
* that have been refilled but not actually consumed yet (not
|
||||
* counting overread bytes, which don't increment 'in_next').
|
||||
*/
|
||||
bitsleft = (u8)bitsleft;
|
||||
SAFETY_CHECK(overread_count <= (bitsleft >> 3));
|
||||
in_next -= (bitsleft >> 3) - overread_count;
|
||||
overread_count = 0;
|
||||
bitbuf = 0;
|
||||
bitsleft = 0;
|
||||
|
||||
SAFETY_CHECK(in_end - in_next >= 4);
|
||||
len = get_unaligned_le16(in_next);
|
||||
nlen = get_unaligned_le16(in_next + 2);
|
||||
in_next += 4;
|
||||
|
||||
SAFETY_CHECK(len == (u16)~nlen);
|
||||
if (unlikely(len > out_end - out_next))
|
||||
return LIBDEFLATE_INSUFFICIENT_SPACE;
|
||||
SAFETY_CHECK(len <= in_end - in_next);
|
||||
|
||||
memcpy(out_next, in_next, len);
|
||||
in_next += len;
|
||||
out_next += len;
|
||||
|
||||
goto block_done;
|
||||
|
||||
} else {
|
||||
unsigned i;
|
||||
|
||||
SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);
|
||||
|
||||
/*
|
||||
* Static Huffman block: build the decode tables for the static
|
||||
* codes. Skip doing so if the tables are already set up from
|
||||
* an earlier static block; this speeds up decompression of
|
||||
* degenerate input of many empty or very short static blocks.
|
||||
*
|
||||
* Afterwards, the remainder is the same as decompressing a
|
||||
* dynamic Huffman block.
|
||||
*/
|
||||
|
||||
bitbuf >>= 3; /* for BTYPE and BFINAL */
|
||||
bitsleft -= 3;
|
||||
|
||||
if (d->static_codes_loaded)
|
||||
goto have_decode_tables;
|
||||
|
||||
d->static_codes_loaded = true;
|
||||
|
||||
STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 288);
|
||||
STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 32);
|
||||
|
||||
for (i = 0; i < 144; i++)
|
||||
d->u.l.lens[i] = 8;
|
||||
for (; i < 256; i++)
|
||||
d->u.l.lens[i] = 9;
|
||||
for (; i < 280; i++)
|
||||
d->u.l.lens[i] = 7;
|
||||
for (; i < 288; i++)
|
||||
d->u.l.lens[i] = 8;
|
||||
|
||||
for (; i < 288 + 32; i++)
|
||||
d->u.l.lens[i] = 5;
|
||||
|
||||
num_litlen_syms = 288;
|
||||
num_offset_syms = 32;
|
||||
}
|
||||
|
||||
/* Decompressing a Huffman block (either dynamic or static) */
|
||||
|
||||
SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));
|
||||
SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));
|
||||
have_decode_tables:
|
||||
litlen_tablemask = BITMASK(d->litlen_tablebits);
|
||||
|
||||
/*
|
||||
* This is the "fastloop" for decoding literals and matches. It does
|
||||
* bounds checks on in_next and out_next in the loop conditions so that
|
||||
* additional bounds checks aren't needed inside the loop body.
|
||||
*
|
||||
* To reduce latency, the bitbuffer is refilled and the next litlen
|
||||
* decode table entry is preloaded before each loop iteration.
|
||||
*/
|
||||
if (in_next >= in_fastloop_end || out_next >= out_fastloop_end)
|
||||
goto generic_loop;
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
|
||||
do {
|
||||
u32 length, offset, lit;
|
||||
const u8 *src;
|
||||
u8 *dst;
|
||||
|
||||
/*
|
||||
* Consume the bits for the litlen decode table entry. Save the
|
||||
* original bitbuf for later, in case the extra match length
|
||||
* bits need to be extracted from it.
|
||||
*/
|
||||
saved_bitbuf = bitbuf;
|
||||
bitbuf >>= (u8)entry;
|
||||
bitsleft -= entry; /* optimization: subtract full entry */
|
||||
|
||||
/*
|
||||
* Begin by checking for a "fast" literal, i.e. a literal that
|
||||
* doesn't need a subtable.
|
||||
*/
|
||||
if (entry & HUFFDEC_LITERAL) {
|
||||
/*
|
||||
* On 64-bit platforms, we decode up to 2 extra fast
|
||||
* literals in addition to the primary item, as this
|
||||
* increases performance and still leaves enough bits
|
||||
* remaining for what follows. We could actually do 3,
|
||||
* assuming LITLEN_TABLEBITS=11, but that actually
|
||||
* decreases performance slightly (perhaps by messing
|
||||
* with the branch prediction of the conditional refill
|
||||
* that happens later while decoding the match offset).
|
||||
*
|
||||
* Note: the definitions of FASTLOOP_MAX_BYTES_WRITTEN
|
||||
* and FASTLOOP_MAX_BYTES_READ need to be updated if the
|
||||
* number of extra literals decoded here is changed.
|
||||
*/
|
||||
if (/* enough bits for 2 fast literals + length + offset preload? */
|
||||
CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
|
||||
LENGTH_MAXBITS,
|
||||
OFFSET_TABLEBITS) &&
|
||||
/* enough bits for 2 fast literals + slow literal + litlen preload? */
|
||||
CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
|
||||
DEFLATE_MAX_LITLEN_CODEWORD_LEN,
|
||||
LITLEN_TABLEBITS)) {
|
||||
/* 1st extra fast literal */
|
||||
lit = entry >> 16;
|
||||
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
|
||||
saved_bitbuf = bitbuf;
|
||||
bitbuf >>= (u8)entry;
|
||||
bitsleft -= entry;
|
||||
*out_next++ = lit;
|
||||
if (entry & HUFFDEC_LITERAL) {
|
||||
/* 2nd extra fast literal */
|
||||
lit = entry >> 16;
|
||||
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
|
||||
saved_bitbuf = bitbuf;
|
||||
bitbuf >>= (u8)entry;
|
||||
bitsleft -= entry;
|
||||
*out_next++ = lit;
|
||||
if (entry & HUFFDEC_LITERAL) {
|
||||
/*
|
||||
* Another fast literal, but
|
||||
* this one is in lieu of the
|
||||
* primary item, so it doesn't
|
||||
* count as one of the extras.
|
||||
*/
|
||||
lit = entry >> 16;
|
||||
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
*out_next++ = lit;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
* Decode a literal. While doing so, preload
|
||||
* the next litlen decode table entry and refill
|
||||
* the bitbuffer. To reduce latency, we've
|
||||
* arranged for there to be enough "preloadable"
|
||||
* bits remaining to do the table preload
|
||||
* independently of the refill.
|
||||
*/
|
||||
STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(
|
||||
LITLEN_TABLEBITS, LITLEN_TABLEBITS));
|
||||
lit = entry >> 16;
|
||||
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
*out_next++ = lit;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* It's not a literal entry, so it can be a length entry, a
|
||||
* subtable pointer entry, or an end-of-block entry. Detect the
|
||||
* two unlikely cases by testing the HUFFDEC_EXCEPTIONAL flag.
|
||||
*/
|
||||
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
|
||||
/* Subtable pointer or end-of-block entry */
|
||||
|
||||
if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
|
||||
goto block_done;
|
||||
|
||||
/*
|
||||
* A subtable is required. Load and consume the
|
||||
* subtable entry. The subtable entry can be of any
|
||||
* type: literal, length, or end-of-block.
|
||||
*/
|
||||
entry = d->u.litlen_decode_table[(entry >> 16) +
|
||||
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
|
||||
saved_bitbuf = bitbuf;
|
||||
bitbuf >>= (u8)entry;
|
||||
bitsleft -= entry;
|
||||
|
||||
/*
|
||||
* 32-bit platforms that use the byte-at-a-time refill
|
||||
* method have to do a refill here for there to always
|
||||
* be enough bits to decode a literal that requires a
|
||||
* subtable, then preload the next litlen decode table
|
||||
* entry; or to decode a match length that requires a
|
||||
* subtable, then preload the offset decode table entry.
|
||||
*/
|
||||
if (!CAN_CONSUME_AND_THEN_PRELOAD(DEFLATE_MAX_LITLEN_CODEWORD_LEN,
|
||||
LITLEN_TABLEBITS) ||
|
||||
!CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXBITS,
|
||||
OFFSET_TABLEBITS))
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
if (entry & HUFFDEC_LITERAL) {
|
||||
/* Decode a literal that required a subtable. */
|
||||
lit = entry >> 16;
|
||||
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
*out_next++ = lit;
|
||||
continue;
|
||||
}
|
||||
if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
|
||||
goto block_done;
|
||||
/* Else, it's a length that required a subtable. */
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode the match length: the length base value associated
|
||||
* with the litlen symbol (which we extract from the decode
|
||||
* table entry), plus the extra length bits. We don't need to
|
||||
* consume the extra length bits here, as they were included in
|
||||
* the bits consumed by the entry earlier. We also don't need
|
||||
* to check for too-long matches here, as this is inside the
|
||||
* fastloop where it's already been verified that the output
|
||||
* buffer has enough space remaining to copy a max-length match.
|
||||
*/
|
||||
length = entry >> 16;
|
||||
length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
|
||||
|
||||
/*
|
||||
* Decode the match offset. There are enough "preloadable" bits
|
||||
* remaining to preload the offset decode table entry, but a
|
||||
* refill might be needed before consuming it.
|
||||
*/
|
||||
STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXFASTBITS,
|
||||
OFFSET_TABLEBITS));
|
||||
entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
|
||||
if (CAN_CONSUME_AND_THEN_PRELOAD(OFFSET_MAXBITS,
|
||||
LITLEN_TABLEBITS)) {
|
||||
/*
|
||||
* Decoding a match offset on a 64-bit platform. We may
|
||||
* need to refill once, but then we can decode the whole
|
||||
* offset and preload the next litlen table entry.
|
||||
*/
|
||||
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
|
||||
/* Offset codeword requires a subtable */
|
||||
if (unlikely((u8)bitsleft < OFFSET_MAXBITS +
|
||||
LITLEN_TABLEBITS - PRELOAD_SLACK))
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
bitbuf >>= OFFSET_TABLEBITS;
|
||||
bitsleft -= OFFSET_TABLEBITS;
|
||||
entry = d->offset_decode_table[(entry >> 16) +
|
||||
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
|
||||
} else if (unlikely((u8)bitsleft < OFFSET_MAXFASTBITS +
|
||||
LITLEN_TABLEBITS - PRELOAD_SLACK))
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
} else {
|
||||
/* Decoding a match offset on a 32-bit platform */
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
|
||||
/* Offset codeword requires a subtable */
|
||||
bitbuf >>= OFFSET_TABLEBITS;
|
||||
bitsleft -= OFFSET_TABLEBITS;
|
||||
entry = d->offset_decode_table[(entry >> 16) +
|
||||
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
/* No further refill needed before extra bits */
|
||||
STATIC_ASSERT(CAN_CONSUME(
|
||||
OFFSET_MAXBITS - OFFSET_TABLEBITS));
|
||||
} else {
|
||||
/* No refill needed before extra bits */
|
||||
STATIC_ASSERT(CAN_CONSUME(OFFSET_MAXFASTBITS));
|
||||
}
|
||||
}
|
||||
saved_bitbuf = bitbuf;
|
||||
bitbuf >>= (u8)entry;
|
||||
bitsleft -= entry; /* optimization: subtract full entry */
|
||||
offset = entry >> 16;
|
||||
offset += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
|
||||
|
||||
/* Validate the match offset; needed even in the fastloop. */
|
||||
SAFETY_CHECK(offset <= out_next - (const u8 *)out);
|
||||
src = out_next - offset;
|
||||
dst = out_next;
|
||||
out_next += length;
|
||||
|
||||
/*
|
||||
* Before starting to issue the instructions to copy the match,
|
||||
* refill the bitbuffer and preload the litlen decode table
|
||||
* entry for the next loop iteration. This can increase
|
||||
* performance by allowing the latency of the match copy to
|
||||
* overlap with these other operations. To further reduce
|
||||
* latency, we've arranged for there to be enough bits remaining
|
||||
* to do the table preload independently of the refill, except
|
||||
* on 32-bit platforms using the byte-at-a-time refill method.
|
||||
*/
|
||||
if (!CAN_CONSUME_AND_THEN_PRELOAD(
|
||||
MAX(OFFSET_MAXBITS - OFFSET_TABLEBITS,
|
||||
OFFSET_MAXFASTBITS),
|
||||
LITLEN_TABLEBITS) &&
|
||||
unlikely((u8)bitsleft < LITLEN_TABLEBITS - PRELOAD_SLACK))
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
|
||||
REFILL_BITS_IN_FASTLOOP();
|
||||
|
||||
/*
|
||||
* Copy the match. On most CPUs the fastest method is a
|
||||
* word-at-a-time copy, unconditionally copying about 5 words
|
||||
* since this is enough for most matches without being too much.
|
||||
*
|
||||
* The normal word-at-a-time copy works for offset >= WORDBYTES,
|
||||
* which is most cases. The case of offset == 1 is also common
|
||||
* and is worth optimizing for, since it is just RLE encoding of
|
||||
* the previous byte, which is the result of compressing long
|
||||
* runs of the same byte.
|
||||
*
|
||||
* Writing past the match 'length' is allowed here, since it's
|
||||
* been ensured there is enough output space left for a slight
|
||||
* overrun. FASTLOOP_MAX_BYTES_WRITTEN needs to be updated if
|
||||
* the maximum possible overrun here is changed.
|
||||
*/
|
||||
if (UNALIGNED_ACCESS_IS_FAST && offset >= WORDBYTES) {
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
while (dst < out_next) {
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += WORDBYTES;
|
||||
dst += WORDBYTES;
|
||||
}
|
||||
} else if (UNALIGNED_ACCESS_IS_FAST && offset == 1) {
|
||||
machine_word_t v;
|
||||
|
||||
/*
|
||||
* This part tends to get auto-vectorized, so keep it
|
||||
* copying a multiple of 16 bytes at a time.
|
||||
*/
|
||||
v = (machine_word_t)0x0101010101010101 * src[0];
|
||||
store_word_unaligned(v, dst);
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(v, dst);
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(v, dst);
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(v, dst);
|
||||
dst += WORDBYTES;
|
||||
while (dst < out_next) {
|
||||
store_word_unaligned(v, dst);
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(v, dst);
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(v, dst);
|
||||
dst += WORDBYTES;
|
||||
store_word_unaligned(v, dst);
|
||||
dst += WORDBYTES;
|
||||
}
|
||||
} else if (UNALIGNED_ACCESS_IS_FAST) {
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += offset;
|
||||
dst += offset;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += offset;
|
||||
dst += offset;
|
||||
do {
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += offset;
|
||||
dst += offset;
|
||||
store_word_unaligned(load_word_unaligned(src), dst);
|
||||
src += offset;
|
||||
dst += offset;
|
||||
} while (dst < out_next);
|
||||
} else {
|
||||
*dst++ = *src++;
|
||||
*dst++ = *src++;
|
||||
do {
|
||||
*dst++ = *src++;
|
||||
} while (dst < out_next);
|
||||
}
|
||||
} while (in_next < in_fastloop_end && out_next < out_fastloop_end);
|
||||
|
||||
/*
|
||||
* This is the generic loop for decoding literals and matches. This
|
||||
* handles cases where in_next and out_next are close to the end of
|
||||
* their respective buffers. Usually this loop isn't performance-
|
||||
* critical, as most time is spent in the fastloop above instead. We
|
||||
* therefore omit some optimizations here in favor of smaller code.
|
||||
*/
|
||||
generic_loop:
|
||||
for (;;) {
|
||||
u32 length, offset;
|
||||
const u8 *src;
|
||||
u8 *dst;
|
||||
|
||||
REFILL_BITS();
|
||||
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
|
||||
saved_bitbuf = bitbuf;
|
||||
bitbuf >>= (u8)entry;
|
||||
bitsleft -= entry;
|
||||
if (unlikely(entry & HUFFDEC_SUBTABLE_POINTER)) {
|
||||
entry = d->u.litlen_decode_table[(entry >> 16) +
|
||||
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
|
||||
saved_bitbuf = bitbuf;
|
||||
bitbuf >>= (u8)entry;
|
||||
bitsleft -= entry;
|
||||
}
|
||||
length = entry >> 16;
|
||||
if (entry & HUFFDEC_LITERAL) {
|
||||
if (unlikely(out_next == out_end))
|
||||
return LIBDEFLATE_INSUFFICIENT_SPACE;
|
||||
*out_next++ = length;
|
||||
continue;
|
||||
}
|
||||
if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
|
||||
goto block_done;
|
||||
length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
|
||||
if (unlikely(length > out_end - out_next))
|
||||
return LIBDEFLATE_INSUFFICIENT_SPACE;
|
||||
|
||||
if (!CAN_CONSUME(LENGTH_MAXBITS + OFFSET_MAXBITS))
|
||||
REFILL_BITS();
|
||||
entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
|
||||
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
|
||||
bitbuf >>= OFFSET_TABLEBITS;
|
||||
bitsleft -= OFFSET_TABLEBITS;
|
||||
entry = d->offset_decode_table[(entry >> 16) +
|
||||
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
|
||||
if (!CAN_CONSUME(OFFSET_MAXBITS))
|
||||
REFILL_BITS();
|
||||
}
|
||||
offset = entry >> 16;
|
||||
offset += EXTRACT_VARBITS8(bitbuf, entry) >> (u8)(entry >> 8);
|
||||
bitbuf >>= (u8)entry;
|
||||
bitsleft -= entry;
|
||||
|
||||
SAFETY_CHECK(offset <= out_next - (const u8 *)out);
|
||||
src = out_next - offset;
|
||||
dst = out_next;
|
||||
out_next += length;
|
||||
|
||||
STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
|
||||
*dst++ = *src++;
|
||||
*dst++ = *src++;
|
||||
do {
|
||||
*dst++ = *src++;
|
||||
} while (dst < out_next);
|
||||
}
|
||||
|
||||
block_done:
|
||||
/* Finished decoding a block */
|
||||
|
||||
if (!is_final_block)
|
||||
goto next_block;
|
||||
|
||||
/* That was the last block. */
|
||||
|
||||
bitsleft = (u8)bitsleft;
|
||||
|
||||
/*
|
||||
* If any of the implicit appended zero bytes were consumed (not just
|
||||
* refilled) before hitting end of stream, then the data is bad.
|
||||
*/
|
||||
SAFETY_CHECK(overread_count <= (bitsleft >> 3));
|
||||
|
||||
/* Optionally return the actual number of bytes consumed. */
|
||||
if (actual_in_nbytes_ret) {
|
||||
/* Don't count bytes that were refilled but not consumed. */
|
||||
in_next -= (bitsleft >> 3) - overread_count;
|
||||
|
||||
*actual_in_nbytes_ret = in_next - (u8 *)in;
|
||||
}
|
||||
|
||||
/* Optionally return the actual number of bytes written. */
|
||||
if (actual_out_nbytes_ret) {
|
||||
*actual_out_nbytes_ret = out_next - (u8 *)out;
|
||||
} else {
|
||||
if (out_next != out_end)
|
||||
return LIBDEFLATE_SHORT_OUTPUT;
|
||||
}
|
||||
return LIBDEFLATE_SUCCESS;
|
||||
}
|
||||
|
||||
#undef FUNCNAME
|
||||
#undef ATTRIBUTES
|
||||
#undef EXTRACT_VARBITS
|
||||
#undef EXTRACT_VARBITS8
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* deflate_constants.h - constants for the DEFLATE compression format
|
||||
*/
|
||||
|
||||
#ifndef LIB_DEFLATE_CONSTANTS_H
|
||||
#define LIB_DEFLATE_CONSTANTS_H
|
||||
|
||||
/* Valid block types */
|
||||
#define DEFLATE_BLOCKTYPE_UNCOMPRESSED 0
|
||||
#define DEFLATE_BLOCKTYPE_STATIC_HUFFMAN 1
|
||||
#define DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN 2
|
||||
|
||||
/* Minimum and maximum supported match lengths (in bytes) */
|
||||
#define DEFLATE_MIN_MATCH_LEN 3
|
||||
#define DEFLATE_MAX_MATCH_LEN 258
|
||||
|
||||
/* Maximum supported match offset (in bytes) */
|
||||
#define DEFLATE_MAX_MATCH_OFFSET 32768
|
||||
|
||||
/* log2 of DEFLATE_MAX_MATCH_OFFSET */
|
||||
#define DEFLATE_WINDOW_ORDER 15
|
||||
|
||||
/* Number of symbols in each Huffman code. Note: for the literal/length
|
||||
* and offset codes, these are actually the maximum values; a given block
|
||||
* might use fewer symbols. */
|
||||
#define DEFLATE_NUM_PRECODE_SYMS 19
|
||||
#define DEFLATE_NUM_LITLEN_SYMS 288
|
||||
#define DEFLATE_NUM_OFFSET_SYMS 32
|
||||
|
||||
/* The maximum number of symbols across all codes */
|
||||
#define DEFLATE_MAX_NUM_SYMS 288
|
||||
|
||||
/* Division of symbols in the literal/length code */
|
||||
#define DEFLATE_NUM_LITERALS 256
|
||||
#define DEFLATE_END_OF_BLOCK 256
|
||||
#define DEFLATE_FIRST_LEN_SYM 257
|
||||
|
||||
/* Maximum codeword length, in bits, within each Huffman code */
|
||||
#define DEFLATE_MAX_PRE_CODEWORD_LEN 7
|
||||
#define DEFLATE_MAX_LITLEN_CODEWORD_LEN 15
|
||||
#define DEFLATE_MAX_OFFSET_CODEWORD_LEN 15
|
||||
|
||||
/* The maximum codeword length across all codes */
|
||||
#define DEFLATE_MAX_CODEWORD_LEN 15
|
||||
|
||||
/* Maximum possible overrun when decoding codeword lengths */
|
||||
#define DEFLATE_MAX_LENS_OVERRUN 137
|
||||
|
||||
/*
|
||||
* Maximum number of extra bits that may be required to represent a match
|
||||
* length or offset.
|
||||
*/
|
||||
#define DEFLATE_MAX_EXTRA_LENGTH_BITS 5
|
||||
#define DEFLATE_MAX_EXTRA_OFFSET_BITS 13
|
||||
|
||||
#endif /* LIB_DEFLATE_CONSTANTS_H */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
* lib_common.h - internal header included by all library code
|
||||
*/
|
||||
|
||||
#ifndef LIB_LIB_COMMON_H
|
||||
#define LIB_LIB_COMMON_H
|
||||
|
||||
#ifdef LIBDEFLATE_H
|
||||
/*
|
||||
* When building the library, LIBDEFLATEAPI needs to be defined properly before
|
||||
* including libdeflate.h.
|
||||
*/
|
||||
# error "lib_common.h must always be included before libdeflate.h"
|
||||
#endif
|
||||
|
||||
#if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
|
||||
# define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
|
||||
#elif defined(__GNUC__)
|
||||
# define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
|
||||
#else
|
||||
# define LIBDEFLATE_EXPORT_SYM
|
||||
#endif
|
||||
|
||||
/*
|
||||
* On i386, gcc assumes that the stack is 16-byte aligned at function entry.
|
||||
* However, some compilers (e.g. MSVC) and programming languages (e.g. Delphi)
|
||||
* only guarantee 4-byte alignment when calling functions. This is mainly an
|
||||
* issue on Windows, but it has been seen on Linux too. Work around this ABI
|
||||
* incompatibility by realigning the stack pointer when entering libdeflate.
|
||||
* This prevents crashes in SSE/AVX code.
|
||||
*/
|
||||
#if defined(__GNUC__) && defined(__i386__)
|
||||
# define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
|
||||
#else
|
||||
# define LIBDEFLATE_ALIGN_STACK
|
||||
#endif
|
||||
|
||||
#define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
|
||||
|
||||
#include "../common_defs.h"
|
||||
|
||||
typedef void *(*malloc_func_t)(size_t);
|
||||
typedef void (*free_func_t)(void *);
|
||||
|
||||
extern malloc_func_t libdeflate_default_malloc_func;
|
||||
extern free_func_t libdeflate_default_free_func;
|
||||
|
||||
void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
|
||||
size_t alignment, size_t size);
|
||||
void libdeflate_aligned_free(free_func_t free_func, void *ptr);
|
||||
|
||||
#ifdef FREESTANDING
|
||||
/*
|
||||
* With -ffreestanding, <string.h> may be missing, and we must provide
|
||||
* implementations of memset(), memcpy(), memmove(), and memcmp().
|
||||
* See https://gcc.gnu.org/onlinedocs/gcc/Standards.html
|
||||
*
|
||||
* Also, -ffreestanding disables interpreting calls to these functions as
|
||||
* built-ins. E.g., calling memcpy(&v, p, WORDBYTES) will make a function call,
|
||||
* not be optimized to a single load instruction. For performance reasons we
|
||||
* don't want that. So, declare these functions as macros that expand to the
|
||||
* corresponding built-ins. This approach is recommended in the gcc man page.
|
||||
* We still need the actual function definitions in case gcc calls them.
|
||||
*/
|
||||
void *memset(void *s, int c, size_t n);
|
||||
#define memset(s, c, n) __builtin_memset((s), (c), (n))
|
||||
|
||||
void *memcpy(void *dest, const void *src, size_t n);
|
||||
#define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
|
||||
|
||||
void *memmove(void *dest, const void *src, size_t n);
|
||||
#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
|
||||
|
||||
int memcmp(const void *s1, const void *s2, size_t n);
|
||||
#define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
|
||||
|
||||
#undef LIBDEFLATE_ENABLE_ASSERTIONS
|
||||
#else
|
||||
# include <string.h>
|
||||
/*
|
||||
* To prevent false positive static analyzer warnings, ensure that assertions
|
||||
* are visible to the static analyzer.
|
||||
*/
|
||||
# ifdef __clang_analyzer__
|
||||
# define LIBDEFLATE_ENABLE_ASSERTIONS
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Runtime assertion support. Don't enable this in production builds; it may
|
||||
* hurt performance significantly.
|
||||
*/
|
||||
#ifdef LIBDEFLATE_ENABLE_ASSERTIONS
|
||||
NORETURN void
|
||||
libdeflate_assertion_failed(const char *expr, const char *file, int line);
|
||||
#define ASSERT(expr) { if (unlikely(!(expr))) \
|
||||
libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
|
||||
#else
|
||||
#define ASSERT(expr) (void)(expr)
|
||||
#endif
|
||||
|
||||
#define CONCAT_IMPL(a, b) a##b
|
||||
#define CONCAT(a, b) CONCAT_IMPL(a, b)
|
||||
#define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
|
||||
|
||||
#endif /* LIB_LIB_COMMON_H */
|
||||
@@ -0,0 +1,141 @@
|
||||
/*
|
||||
* utils.c - utility functions for libdeflate
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "lib_common.h"
|
||||
|
||||
#ifdef FREESTANDING
|
||||
# define malloc NULL
|
||||
# define free NULL
|
||||
#else
|
||||
# include <stdlib.h>
|
||||
#endif
|
||||
|
||||
malloc_func_t libdeflate_default_malloc_func = malloc;
|
||||
free_func_t libdeflate_default_free_func = free;
|
||||
|
||||
void *
|
||||
libdeflate_aligned_malloc(malloc_func_t malloc_func,
|
||||
size_t alignment, size_t size)
|
||||
{
|
||||
void *ptr = (*malloc_func)(sizeof(void *) + alignment - 1 + size);
|
||||
|
||||
if (ptr) {
|
||||
void *orig_ptr = ptr;
|
||||
|
||||
ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment);
|
||||
((void **)ptr)[-1] = orig_ptr;
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void
|
||||
libdeflate_aligned_free(free_func_t free_func, void *ptr)
|
||||
{
|
||||
(*free_func)(((void **)ptr)[-1]);
|
||||
}
|
||||
|
||||
LIBDEFLATEAPI void
|
||||
libdeflate_set_memory_allocator(malloc_func_t malloc_func,
|
||||
free_func_t free_func)
|
||||
{
|
||||
libdeflate_default_malloc_func = malloc_func;
|
||||
libdeflate_default_free_func = free_func;
|
||||
}
|
||||
|
||||
/*
|
||||
* Implementations of libc functions for freestanding library builds.
|
||||
* Normal library builds don't use these. Not optimized yet; usually the
|
||||
* compiler expands these functions and doesn't actually call them anyway.
|
||||
*/
|
||||
#ifdef FREESTANDING
|
||||
#undef memset
|
||||
void * __attribute__((weak))
|
||||
memset(void *s, int c, size_t n)
|
||||
{
|
||||
u8 *p = s;
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < n; i++)
|
||||
p[i] = c;
|
||||
return s;
|
||||
}
|
||||
|
||||
#undef memcpy
|
||||
void * __attribute__((weak))
|
||||
memcpy(void *dest, const void *src, size_t n)
|
||||
{
|
||||
u8 *d = dest;
|
||||
const u8 *s = src;
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < n; i++)
|
||||
d[i] = s[i];
|
||||
return dest;
|
||||
}
|
||||
|
||||
#undef memmove
|
||||
void * __attribute__((weak))
|
||||
memmove(void *dest, const void *src, size_t n)
|
||||
{
|
||||
u8 *d = dest;
|
||||
const u8 *s = src;
|
||||
size_t i;
|
||||
|
||||
if (d <= s)
|
||||
return memcpy(d, s, n);
|
||||
|
||||
for (i = n; i > 0; i--)
|
||||
d[i - 1] = s[i - 1];
|
||||
return dest;
|
||||
}
|
||||
|
||||
#undef memcmp
|
||||
int __attribute__((weak))
|
||||
memcmp(const void *s1, const void *s2, size_t n)
|
||||
{
|
||||
const u8 *p1 = s1;
|
||||
const u8 *p2 = s2;
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < n; i++) {
|
||||
if (p1[i] != p2[i])
|
||||
return (int)p1[i] - (int)p2[i];
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif /* FREESTANDING */
|
||||
|
||||
#ifdef LIBDEFLATE_ENABLE_ASSERTIONS
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
NORETURN void
|
||||
libdeflate_assertion_failed(const char *expr, const char *file, int line)
|
||||
{
|
||||
fprintf(stderr, "Assertion failed: %s at %s:%d\n", expr, file, line);
|
||||
abort();
|
||||
}
|
||||
#endif /* LIBDEFLATE_ENABLE_ASSERTIONS */
|
||||
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* x86/adler32_impl.h - x86 implementations of Adler-32 checksum algorithm
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef LIB_X86_ADLER32_IMPL_H
|
||||
#define LIB_X86_ADLER32_IMPL_H
|
||||
|
||||
#include "cpu_features.h"
|
||||
|
||||
/* SSE2 and AVX2 implementations. Used on older CPUs. */
|
||||
#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
|
||||
# define adler32_x86_sse2 adler32_x86_sse2
|
||||
# define SUFFIX _sse2
|
||||
# define ATTRIBUTES _target_attribute("sse2")
|
||||
# define VL 16
|
||||
# define USE_VNNI 0
|
||||
# define USE_AVX512 0
|
||||
# include "adler32_template.h"
|
||||
|
||||
# define adler32_x86_avx2 adler32_x86_avx2
|
||||
# define SUFFIX _avx2
|
||||
# define ATTRIBUTES _target_attribute("avx2")
|
||||
# define VL 32
|
||||
# define USE_VNNI 0
|
||||
# define USE_AVX512 0
|
||||
# include "adler32_template.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* AVX-VNNI implementation. This is used on CPUs that have AVX2 and AVX-VNNI
|
||||
* but don't have AVX-512, for example Intel Alder Lake.
|
||||
*
|
||||
* Unusually for a new CPU feature, gcc added support for the AVX-VNNI
|
||||
* intrinsics (in gcc 11.1) slightly before binutils added support for
|
||||
* assembling AVX-VNNI instructions (in binutils 2.36). Distros can reasonably
|
||||
* have gcc 11 with binutils 2.35. Because of this issue, we check for gcc 12
|
||||
* instead of gcc 11. (libdeflate supports direct compilation without a
|
||||
* configure step, so checking the binutils version is not always an option.)
|
||||
*/
|
||||
#if (GCC_PREREQ(12, 1) || CLANG_PREREQ(12, 0, 13000000) || MSVC_PREREQ(1930)) && \
|
||||
!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX_VNNI)
|
||||
# define adler32_x86_avx2_vnni adler32_x86_avx2_vnni
|
||||
# define SUFFIX _avx2_vnni
|
||||
# define ATTRIBUTES _target_attribute("avx2,avxvnni")
|
||||
# define VL 32
|
||||
# define USE_VNNI 1
|
||||
# define USE_AVX512 0
|
||||
# include "adler32_template.h"
|
||||
#endif
|
||||
|
||||
#if (GCC_PREREQ(8, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
|
||||
!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX512VNNI)
|
||||
/*
|
||||
* AVX512VNNI implementation using 256-bit vectors. This is very similar to the
|
||||
* AVX-VNNI implementation but takes advantage of masking and more registers.
|
||||
* This is used on certain older Intel CPUs, specifically Ice Lake and Tiger
|
||||
* Lake, which support AVX512VNNI but downclock a bit too eagerly when ZMM
|
||||
* registers are used.
|
||||
*/
|
||||
# define adler32_x86_avx512_vl256_vnni adler32_x86_avx512_vl256_vnni
|
||||
# define SUFFIX _avx512_vl256_vnni
|
||||
# define ATTRIBUTES _target_attribute("avx512bw,avx512vl,avx512vnni")
|
||||
# define VL 32
|
||||
# define USE_VNNI 1
|
||||
# define USE_AVX512 1
|
||||
# include "adler32_template.h"
|
||||
|
||||
/*
|
||||
* AVX512VNNI implementation using 512-bit vectors. This is used on CPUs that
|
||||
* have a good AVX-512 implementation including AVX512VNNI.
|
||||
*/
|
||||
# define adler32_x86_avx512_vl512_vnni adler32_x86_avx512_vl512_vnni
|
||||
# define SUFFIX _avx512_vl512_vnni
|
||||
# define ATTRIBUTES _target_attribute("avx512bw,avx512vnni")
|
||||
# define VL 64
|
||||
# define USE_VNNI 1
|
||||
# define USE_AVX512 1
|
||||
# include "adler32_template.h"
|
||||
#endif
|
||||
|
||||
static inline adler32_func_t
|
||||
arch_select_adler32_func(void)
|
||||
{
|
||||
const u32 features MAYBE_UNUSED = get_x86_cpu_features();
|
||||
|
||||
#ifdef adler32_x86_avx512_vl512_vnni
|
||||
if ((features & X86_CPU_FEATURE_ZMM) &&
|
||||
HAVE_AVX512BW(features) && HAVE_AVX512VNNI(features))
|
||||
return adler32_x86_avx512_vl512_vnni;
|
||||
#endif
|
||||
#ifdef adler32_x86_avx512_vl256_vnni
|
||||
if (HAVE_AVX512BW(features) && HAVE_AVX512VL(features) &&
|
||||
HAVE_AVX512VNNI(features))
|
||||
return adler32_x86_avx512_vl256_vnni;
|
||||
#endif
|
||||
#ifdef adler32_x86_avx2_vnni
|
||||
if (HAVE_AVX2(features) && HAVE_AVXVNNI(features))
|
||||
return adler32_x86_avx2_vnni;
|
||||
#endif
|
||||
#ifdef adler32_x86_avx2
|
||||
if (HAVE_AVX2(features))
|
||||
return adler32_x86_avx2;
|
||||
#endif
|
||||
#ifdef adler32_x86_sse2
|
||||
if (HAVE_SSE2(features))
|
||||
return adler32_x86_sse2;
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
#define arch_select_adler32_func arch_select_adler32_func
|
||||
|
||||
#endif /* LIB_X86_ADLER32_IMPL_H */
|
||||
@@ -0,0 +1,518 @@
|
||||
/*
|
||||
* x86/adler32_template.h - template for vectorized Adler-32 implementations
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is a "template" for instantiating Adler-32 functions for x86.
|
||||
* The "parameters" are:
|
||||
*
|
||||
* SUFFIX:
|
||||
* Name suffix to append to all instantiated functions.
|
||||
* ATTRIBUTES:
|
||||
* Target function attributes to use. Must satisfy the dependencies of the
|
||||
* other parameters as follows:
|
||||
* VL=16 && USE_VNNI=0 && USE_AVX512=0: at least sse2
|
||||
* VL=32 && USE_VNNI=0 && USE_AVX512=0: at least avx2
|
||||
* VL=32 && USE_VNNI=1 && USE_AVX512=0: at least avx2,avxvnni
|
||||
* VL=32 && USE_VNNI=1 && USE_AVX512=1: at least avx512bw,avx512vl,avx512vnni
|
||||
* VL=64 && USE_VNNI=1 && USE_AVX512=1: at least avx512bw,avx512vnni
|
||||
* (Other combinations are not useful and have not been tested.)
|
||||
* VL:
|
||||
* Vector length in bytes. Must be 16, 32, or 64.
|
||||
* USE_VNNI:
|
||||
* If 1, use the VNNI dot product based algorithm.
|
||||
* If 0, use the legacy SSE2 and AVX2 compatible algorithm.
|
||||
* USE_AVX512:
|
||||
* If 1, take advantage of AVX-512 features such as masking. This doesn't
|
||||
* enable the use of 512-bit vectors; the vector length is controlled by
|
||||
* VL. If 0, assume that the CPU might not support AVX-512.
|
||||
*/
|
||||
|
||||
#if VL == 16
|
||||
# define vec_t __m128i
|
||||
# define mask_t u16
|
||||
# define LOG2_VL 4
|
||||
# define VADD8(a, b) _mm_add_epi8((a), (b))
|
||||
# define VADD16(a, b) _mm_add_epi16((a), (b))
|
||||
# define VADD32(a, b) _mm_add_epi32((a), (b))
|
||||
# if USE_AVX512
|
||||
# define VDPBUSD(a, b, c) _mm_dpbusd_epi32((a), (b), (c))
|
||||
# else
|
||||
# define VDPBUSD(a, b, c) _mm_dpbusd_avx_epi32((a), (b), (c))
|
||||
# endif
|
||||
# define VLOAD(p) _mm_load_si128((const void *)(p))
|
||||
# define VLOADU(p) _mm_loadu_si128((const void *)(p))
|
||||
# define VMADD16(a, b) _mm_madd_epi16((a), (b))
|
||||
# define VMASKZ_LOADU(mask, p) _mm_maskz_loadu_epi8((mask), (p))
|
||||
# define VMULLO32(a, b) _mm_mullo_epi32((a), (b))
|
||||
# define VSAD8(a, b) _mm_sad_epu8((a), (b))
|
||||
# define VSET1_8(a) _mm_set1_epi8(a)
|
||||
# define VSET1_32(a) _mm_set1_epi32(a)
|
||||
# define VSETZERO() _mm_setzero_si128()
|
||||
# define VSLL32(a, b) _mm_slli_epi32((a), (b))
|
||||
# define VUNPACKLO8(a, b) _mm_unpacklo_epi8((a), (b))
|
||||
# define VUNPACKHI8(a, b) _mm_unpackhi_epi8((a), (b))
|
||||
#elif VL == 32
|
||||
# define vec_t __m256i
|
||||
# define mask_t u32
|
||||
# define LOG2_VL 5
|
||||
# define VADD8(a, b) _mm256_add_epi8((a), (b))
|
||||
# define VADD16(a, b) _mm256_add_epi16((a), (b))
|
||||
# define VADD32(a, b) _mm256_add_epi32((a), (b))
|
||||
# if USE_AVX512
|
||||
# define VDPBUSD(a, b, c) _mm256_dpbusd_epi32((a), (b), (c))
|
||||
# else
|
||||
# define VDPBUSD(a, b, c) _mm256_dpbusd_avx_epi32((a), (b), (c))
|
||||
# endif
|
||||
# define VLOAD(p) _mm256_load_si256((const void *)(p))
|
||||
# define VLOADU(p) _mm256_loadu_si256((const void *)(p))
|
||||
# define VMADD16(a, b) _mm256_madd_epi16((a), (b))
|
||||
# define VMASKZ_LOADU(mask, p) _mm256_maskz_loadu_epi8((mask), (p))
|
||||
# define VMULLO32(a, b) _mm256_mullo_epi32((a), (b))
|
||||
# define VSAD8(a, b) _mm256_sad_epu8((a), (b))
|
||||
# define VSET1_8(a) _mm256_set1_epi8(a)
|
||||
# define VSET1_32(a) _mm256_set1_epi32(a)
|
||||
# define VSETZERO() _mm256_setzero_si256()
|
||||
# define VSLL32(a, b) _mm256_slli_epi32((a), (b))
|
||||
# define VUNPACKLO8(a, b) _mm256_unpacklo_epi8((a), (b))
|
||||
# define VUNPACKHI8(a, b) _mm256_unpackhi_epi8((a), (b))
|
||||
#elif VL == 64
|
||||
# define vec_t __m512i
|
||||
# define mask_t u64
|
||||
# define LOG2_VL 6
|
||||
# define VADD8(a, b) _mm512_add_epi8((a), (b))
|
||||
# define VADD16(a, b) _mm512_add_epi16((a), (b))
|
||||
# define VADD32(a, b) _mm512_add_epi32((a), (b))
|
||||
# define VDPBUSD(a, b, c) _mm512_dpbusd_epi32((a), (b), (c))
|
||||
# define VLOAD(p) _mm512_load_si512((const void *)(p))
|
||||
# define VLOADU(p) _mm512_loadu_si512((const void *)(p))
|
||||
# define VMADD16(a, b) _mm512_madd_epi16((a), (b))
|
||||
# define VMASKZ_LOADU(mask, p) _mm512_maskz_loadu_epi8((mask), (p))
|
||||
# define VMULLO32(a, b) _mm512_mullo_epi32((a), (b))
|
||||
# define VSAD8(a, b) _mm512_sad_epu8((a), (b))
|
||||
# define VSET1_8(a) _mm512_set1_epi8(a)
|
||||
# define VSET1_32(a) _mm512_set1_epi32(a)
|
||||
# define VSETZERO() _mm512_setzero_si512()
|
||||
# define VSLL32(a, b) _mm512_slli_epi32((a), (b))
|
||||
# define VUNPACKLO8(a, b) _mm512_unpacklo_epi8((a), (b))
|
||||
# define VUNPACKHI8(a, b) _mm512_unpackhi_epi8((a), (b))
|
||||
#else
|
||||
# error "unsupported vector length"
|
||||
#endif
|
||||
|
||||
#define VADD32_3X(a, b, c) VADD32(VADD32((a), (b)), (c))
|
||||
#define VADD32_4X(a, b, c, d) VADD32(VADD32((a), (b)), VADD32((c), (d)))
|
||||
#define VADD32_5X(a, b, c, d, e) VADD32((a), VADD32_4X((b), (c), (d), (e)))
|
||||
#define VADD32_7X(a, b, c, d, e, f, g) \
|
||||
VADD32(VADD32_3X((a), (b), (c)), VADD32_4X((d), (e), (f), (g)))
|
||||
|
||||
/* Sum the 32-bit elements of v_s1 and add them to s1, and likewise for s2. */
|
||||
#undef reduce_to_32bits
|
||||
static forceinline ATTRIBUTES void
|
||||
ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
|
||||
{
|
||||
__m128i v_s1_128, v_s2_128;
|
||||
#if VL == 16
|
||||
{
|
||||
v_s1_128 = v_s1;
|
||||
v_s2_128 = v_s2;
|
||||
}
|
||||
#else
|
||||
{
|
||||
__m256i v_s1_256, v_s2_256;
|
||||
#if VL == 32
|
||||
v_s1_256 = v_s1;
|
||||
v_s2_256 = v_s2;
|
||||
#else
|
||||
/* Reduce 512 bits to 256 bits. */
|
||||
v_s1_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s1, 0),
|
||||
_mm512_extracti64x4_epi64(v_s1, 1));
|
||||
v_s2_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s2, 0),
|
||||
_mm512_extracti64x4_epi64(v_s2, 1));
|
||||
#endif
|
||||
/* Reduce 256 bits to 128 bits. */
|
||||
v_s1_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s1_256, 0),
|
||||
_mm256_extracti128_si256(v_s1_256, 1));
|
||||
v_s2_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s2_256, 0),
|
||||
_mm256_extracti128_si256(v_s2_256, 1));
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Reduce 128 bits to 32 bits.
|
||||
*
|
||||
* If the bytes were summed into v_s1 using psadbw + paddd, then ignore
|
||||
* the odd-indexed elements of v_s1_128 since they are zero.
|
||||
*/
|
||||
#if USE_VNNI
|
||||
v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x31));
|
||||
#endif
|
||||
v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x31));
|
||||
v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x02));
|
||||
v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x02));
|
||||
|
||||
*s1_p += (u32)_mm_cvtsi128_si32(v_s1_128);
|
||||
*s2_p += (u32)_mm_cvtsi128_si32(v_s2_128);
|
||||
}
|
||||
#define reduce_to_32bits ADD_SUFFIX(reduce_to_32bits)
|
||||
|
||||
static ATTRIBUTES u32
|
||||
ADD_SUFFIX(adler32_x86)(u32 adler, const u8 *p, size_t len)
|
||||
{
|
||||
#if USE_VNNI
|
||||
/* This contains the bytes [VL, VL-1, VL-2, ..., 1]. */
|
||||
static const u8 _aligned_attribute(VL) raw_mults[VL] = {
|
||||
#if VL == 64
|
||||
64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
|
||||
48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
|
||||
#endif
|
||||
#if VL >= 32
|
||||
32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
|
||||
#endif
|
||||
16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
|
||||
};
|
||||
const vec_t ones = VSET1_8(1);
|
||||
#else
|
||||
/*
|
||||
* This contains the 16-bit values [2*VL, 2*VL - 1, 2*VL - 2, ..., 1].
|
||||
* For VL==32 the ordering is weird because it has to match the way that
|
||||
* vpunpcklbw and vpunpckhbw work on 128-bit lanes separately.
|
||||
*/
|
||||
static const u16 _aligned_attribute(VL) raw_mults[4][VL / 2] = {
|
||||
#if VL == 16
|
||||
{ 32, 31, 30, 29, 28, 27, 26, 25 },
|
||||
{ 24, 23, 22, 21, 20, 19, 18, 17 },
|
||||
{ 16, 15, 14, 13, 12, 11, 10, 9 },
|
||||
{ 8, 7, 6, 5, 4, 3, 2, 1 },
|
||||
#elif VL == 32
|
||||
{ 64, 63, 62, 61, 60, 59, 58, 57, 48, 47, 46, 45, 44, 43, 42, 41 },
|
||||
{ 56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33 },
|
||||
{ 32, 31, 30, 29, 28, 27, 26, 25, 16, 15, 14, 13, 12, 11, 10, 9 },
|
||||
{ 24, 23, 22, 21, 20, 19, 18, 17, 8, 7, 6, 5, 4, 3, 2, 1 },
|
||||
#else
|
||||
# error "unsupported parameters"
|
||||
#endif
|
||||
};
|
||||
const vec_t mults_a = VLOAD(raw_mults[0]);
|
||||
const vec_t mults_b = VLOAD(raw_mults[1]);
|
||||
const vec_t mults_c = VLOAD(raw_mults[2]);
|
||||
const vec_t mults_d = VLOAD(raw_mults[3]);
|
||||
#endif
|
||||
const vec_t zeroes = VSETZERO();
|
||||
u32 s1 = adler & 0xFFFF;
|
||||
u32 s2 = adler >> 16;
|
||||
|
||||
/*
|
||||
* If the length is large and the pointer is misaligned, align it.
|
||||
* For smaller lengths, just take the misaligned load penalty.
|
||||
*/
|
||||
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) {
|
||||
do {
|
||||
s1 += *p++;
|
||||
s2 += s1;
|
||||
len--;
|
||||
} while ((uintptr_t)p & (VL-1));
|
||||
s1 %= DIVISOR;
|
||||
s2 %= DIVISOR;
|
||||
}
|
||||
|
||||
#if USE_VNNI
|
||||
/*
|
||||
* This is Adler-32 using the vpdpbusd instruction from AVX512VNNI or
|
||||
* AVX-VNNI. vpdpbusd multiplies the unsigned bytes of one vector by
|
||||
* the signed bytes of another vector and adds the sums in groups of 4
|
||||
* to the 32-bit elements of a third vector. We use it in two ways:
|
||||
* multiplying the data bytes by a sequence like 64,63,62,...,1 for
|
||||
* calculating part of s2, and multiplying the data bytes by an all-ones
|
||||
* sequence 1,1,1,...,1 for calculating s1 and part of s2. The all-ones
|
||||
* trick seems to be faster than the alternative of vpsadbw + vpaddd.
|
||||
*/
|
||||
while (len) {
|
||||
/*
|
||||
* Calculate the length of the next data chunk such that s1 and
|
||||
* s2 are guaranteed to not exceed UINT32_MAX.
|
||||
*/
|
||||
size_t n = MIN(len, MAX_CHUNK_LEN & ~(4*VL - 1));
|
||||
vec_t mults = VLOAD(raw_mults);
|
||||
vec_t v_s1 = zeroes;
|
||||
vec_t v_s2 = zeroes;
|
||||
|
||||
s2 += s1 * n;
|
||||
len -= n;
|
||||
|
||||
if (n >= 4*VL) {
|
||||
vec_t v_s1_b = zeroes;
|
||||
vec_t v_s1_c = zeroes;
|
||||
vec_t v_s1_d = zeroes;
|
||||
vec_t v_s2_b = zeroes;
|
||||
vec_t v_s2_c = zeroes;
|
||||
vec_t v_s2_d = zeroes;
|
||||
vec_t v_s1_sums = zeroes;
|
||||
vec_t v_s1_sums_b = zeroes;
|
||||
vec_t v_s1_sums_c = zeroes;
|
||||
vec_t v_s1_sums_d = zeroes;
|
||||
vec_t tmp0, tmp1;
|
||||
|
||||
do {
|
||||
vec_t data_a = VLOADU(p + 0*VL);
|
||||
vec_t data_b = VLOADU(p + 1*VL);
|
||||
vec_t data_c = VLOADU(p + 2*VL);
|
||||
vec_t data_d = VLOADU(p + 3*VL);
|
||||
|
||||
/*
|
||||
* Workaround for gcc bug where it generates
|
||||
* unnecessary move instructions
|
||||
* (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107892)
|
||||
*/
|
||||
#if GCC_PREREQ(1, 0)
|
||||
__asm__("" : "+v" (data_a), "+v" (data_b),
|
||||
"+v" (data_c), "+v" (data_d));
|
||||
#endif
|
||||
|
||||
v_s2 = VDPBUSD(v_s2, data_a, mults);
|
||||
v_s2_b = VDPBUSD(v_s2_b, data_b, mults);
|
||||
v_s2_c = VDPBUSD(v_s2_c, data_c, mults);
|
||||
v_s2_d = VDPBUSD(v_s2_d, data_d, mults);
|
||||
|
||||
v_s1_sums = VADD32(v_s1_sums, v_s1);
|
||||
v_s1_sums_b = VADD32(v_s1_sums_b, v_s1_b);
|
||||
v_s1_sums_c = VADD32(v_s1_sums_c, v_s1_c);
|
||||
v_s1_sums_d = VADD32(v_s1_sums_d, v_s1_d);
|
||||
|
||||
v_s1 = VDPBUSD(v_s1, data_a, ones);
|
||||
v_s1_b = VDPBUSD(v_s1_b, data_b, ones);
|
||||
v_s1_c = VDPBUSD(v_s1_c, data_c, ones);
|
||||
v_s1_d = VDPBUSD(v_s1_d, data_d, ones);
|
||||
|
||||
/* Same gcc bug workaround. See above */
|
||||
#if GCC_PREREQ(1, 0) && !defined(ARCH_X86_32)
|
||||
__asm__("" : "+v" (v_s2), "+v" (v_s2_b),
|
||||
"+v" (v_s2_c), "+v" (v_s2_d),
|
||||
"+v" (v_s1_sums),
|
||||
"+v" (v_s1_sums_b),
|
||||
"+v" (v_s1_sums_c),
|
||||
"+v" (v_s1_sums_d),
|
||||
"+v" (v_s1), "+v" (v_s1_b),
|
||||
"+v" (v_s1_c), "+v" (v_s1_d));
|
||||
#endif
|
||||
p += 4*VL;
|
||||
n -= 4*VL;
|
||||
} while (n >= 4*VL);
|
||||
|
||||
/*
|
||||
* Reduce into v_s1 and v_s2 as follows:
|
||||
*
|
||||
* v_s2 = v_s2 + v_s2_b + v_s2_c + v_s2_d +
|
||||
* (4*VL)*(v_s1_sums + v_s1_sums_b +
|
||||
* v_s1_sums_c + v_s1_sums_d) +
|
||||
* (3*VL)*v_s1 + (2*VL)*v_s1_b + VL*v_s1_c
|
||||
* v_s1 = v_s1 + v_s1_b + v_s1_c + v_s1_d
|
||||
*/
|
||||
tmp0 = VADD32(v_s1, v_s1_b);
|
||||
tmp1 = VADD32(v_s1, v_s1_c);
|
||||
v_s1_sums = VADD32_4X(v_s1_sums, v_s1_sums_b,
|
||||
v_s1_sums_c, v_s1_sums_d);
|
||||
v_s1 = VADD32_3X(tmp0, v_s1_c, v_s1_d);
|
||||
v_s2 = VADD32_7X(VSLL32(v_s1_sums, LOG2_VL + 2),
|
||||
VSLL32(tmp0, LOG2_VL + 1),
|
||||
VSLL32(tmp1, LOG2_VL),
|
||||
v_s2, v_s2_b, v_s2_c, v_s2_d);
|
||||
}
|
||||
|
||||
/* Process the last 0 <= n < 4*VL bytes of the chunk. */
|
||||
if (n >= 2*VL) {
|
||||
const vec_t data_a = VLOADU(p + 0*VL);
|
||||
const vec_t data_b = VLOADU(p + 1*VL);
|
||||
|
||||
v_s2 = VADD32(v_s2, VSLL32(v_s1, LOG2_VL + 1));
|
||||
v_s1 = VDPBUSD(v_s1, data_a, ones);
|
||||
v_s1 = VDPBUSD(v_s1, data_b, ones);
|
||||
v_s2 = VDPBUSD(v_s2, data_a, VSET1_8(VL));
|
||||
v_s2 = VDPBUSD(v_s2, data_a, mults);
|
||||
v_s2 = VDPBUSD(v_s2, data_b, mults);
|
||||
p += 2*VL;
|
||||
n -= 2*VL;
|
||||
}
|
||||
if (n) {
|
||||
/* Process the last 0 < n < 2*VL bytes of the chunk. */
|
||||
vec_t data;
|
||||
|
||||
v_s2 = VADD32(v_s2, VMULLO32(v_s1, VSET1_32(n)));
|
||||
|
||||
mults = VADD8(mults, VSET1_8((int)n - VL));
|
||||
if (n > VL) {
|
||||
data = VLOADU(p);
|
||||
v_s1 = VDPBUSD(v_s1, data, ones);
|
||||
v_s2 = VDPBUSD(v_s2, data, mults);
|
||||
p += VL;
|
||||
n -= VL;
|
||||
mults = VADD8(mults, VSET1_8(-VL));
|
||||
}
|
||||
/*
|
||||
* Process the last 0 < n <= VL bytes of the chunk.
|
||||
* Utilize a masked load if it's available.
|
||||
*/
|
||||
#if USE_AVX512
|
||||
data = VMASKZ_LOADU((mask_t)-1 >> (VL - n), p);
|
||||
#else
|
||||
data = zeroes;
|
||||
memcpy(&data, p, n);
|
||||
#endif
|
||||
v_s1 = VDPBUSD(v_s1, data, ones);
|
||||
v_s2 = VDPBUSD(v_s2, data, mults);
|
||||
p += n;
|
||||
}
|
||||
|
||||
reduce_to_32bits(v_s1, v_s2, &s1, &s2);
|
||||
s1 %= DIVISOR;
|
||||
s2 %= DIVISOR;
|
||||
}
|
||||
#else /* USE_VNNI */
|
||||
/*
|
||||
* This is Adler-32 for SSE2 and AVX2.
|
||||
*
|
||||
* To horizontally sum bytes, use psadbw + paddd, where one of the
|
||||
* arguments to psadbw is all-zeroes.
|
||||
*
|
||||
* For the s2 contribution from (2*VL - i)*data[i] for each of the 2*VL
|
||||
* bytes of each iteration of the inner loop, use punpck{l,h}bw + paddw
|
||||
* to sum, for each i across iterations, byte i into a corresponding
|
||||
* 16-bit counter in v_byte_sums_*. After the inner loop, use pmaddwd
|
||||
* to multiply each counter by (2*VL - i), then add the products to s2.
|
||||
*
|
||||
* An alternative implementation would use pmaddubsw and pmaddwd in the
|
||||
* inner loop to do (2*VL - i)*data[i] directly and add the products in
|
||||
* groups of 4 to 32-bit counters. However, on average that approach
|
||||
* seems to be slower than the current approach which delays the
|
||||
* multiplications. Also, pmaddubsw requires SSSE3; the current
|
||||
* approach keeps the implementation aligned between SSE2 and AVX2.
|
||||
*
|
||||
* The inner loop processes 2*VL bytes per iteration. Increasing this
|
||||
* to 4*VL doesn't seem to be helpful here.
|
||||
*/
|
||||
while (len) {
|
||||
/*
|
||||
* Calculate the length of the next data chunk such that s1 and
|
||||
* s2 are guaranteed to not exceed UINT32_MAX, and every
|
||||
* v_byte_sums_* counter is guaranteed to not exceed INT16_MAX.
|
||||
* It's INT16_MAX, not UINT16_MAX, because v_byte_sums_* are
|
||||
* used with pmaddwd which does signed multiplication. In the
|
||||
* SSE2 case this limits chunks to 4096 bytes instead of 5536.
|
||||
*/
|
||||
size_t n = MIN(len, MIN(2 * VL * (INT16_MAX / UINT8_MAX),
|
||||
MAX_CHUNK_LEN) & ~(2*VL - 1));
|
||||
len -= n;
|
||||
|
||||
if (n >= 2*VL) {
|
||||
vec_t v_s1 = zeroes;
|
||||
vec_t v_s1_sums = zeroes;
|
||||
vec_t v_byte_sums_a = zeroes;
|
||||
vec_t v_byte_sums_b = zeroes;
|
||||
vec_t v_byte_sums_c = zeroes;
|
||||
vec_t v_byte_sums_d = zeroes;
|
||||
vec_t v_s2;
|
||||
|
||||
s2 += s1 * (n & ~(2*VL - 1));
|
||||
|
||||
do {
|
||||
vec_t data_a = VLOADU(p + 0*VL);
|
||||
vec_t data_b = VLOADU(p + 1*VL);
|
||||
|
||||
v_s1_sums = VADD32(v_s1_sums, v_s1);
|
||||
v_byte_sums_a = VADD16(v_byte_sums_a,
|
||||
VUNPACKLO8(data_a, zeroes));
|
||||
v_byte_sums_b = VADD16(v_byte_sums_b,
|
||||
VUNPACKHI8(data_a, zeroes));
|
||||
v_byte_sums_c = VADD16(v_byte_sums_c,
|
||||
VUNPACKLO8(data_b, zeroes));
|
||||
v_byte_sums_d = VADD16(v_byte_sums_d,
|
||||
VUNPACKHI8(data_b, zeroes));
|
||||
v_s1 = VADD32(v_s1,
|
||||
VADD32(VSAD8(data_a, zeroes),
|
||||
VSAD8(data_b, zeroes)));
|
||||
/*
|
||||
* Workaround for gcc bug where it generates
|
||||
* unnecessary move instructions
|
||||
* (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107892)
|
||||
*/
|
||||
#if GCC_PREREQ(1, 0)
|
||||
__asm__("" : "+x" (v_s1), "+x" (v_s1_sums),
|
||||
"+x" (v_byte_sums_a),
|
||||
"+x" (v_byte_sums_b),
|
||||
"+x" (v_byte_sums_c),
|
||||
"+x" (v_byte_sums_d));
|
||||
#endif
|
||||
p += 2*VL;
|
||||
n -= 2*VL;
|
||||
} while (n >= 2*VL);
|
||||
|
||||
/*
|
||||
* Calculate v_s2 as (2*VL)*v_s1_sums +
|
||||
* [2*VL, 2*VL - 1, 2*VL - 2, ..., 1] * v_byte_sums.
|
||||
* Then update s1 and s2 from v_s1 and v_s2.
|
||||
*/
|
||||
v_s2 = VADD32_5X(VSLL32(v_s1_sums, LOG2_VL + 1),
|
||||
VMADD16(v_byte_sums_a, mults_a),
|
||||
VMADD16(v_byte_sums_b, mults_b),
|
||||
VMADD16(v_byte_sums_c, mults_c),
|
||||
VMADD16(v_byte_sums_d, mults_d));
|
||||
reduce_to_32bits(v_s1, v_s2, &s1, &s2);
|
||||
}
|
||||
/*
|
||||
* Process the last 0 <= n < 2*VL bytes of the chunk using
|
||||
* scalar instructions and reduce s1 and s2 mod DIVISOR.
|
||||
*/
|
||||
ADLER32_CHUNK(s1, s2, p, n);
|
||||
}
|
||||
#endif /* !USE_VNNI */
|
||||
return (s2 << 16) | s1;
|
||||
}
|
||||
|
||||
#undef vec_t
|
||||
#undef mask_t
|
||||
#undef LOG2_VL
|
||||
#undef VADD8
|
||||
#undef VADD16
|
||||
#undef VADD32
|
||||
#undef VDPBUSD
|
||||
#undef VLOAD
|
||||
#undef VLOADU
|
||||
#undef VMADD16
|
||||
#undef VMASKZ_LOADU
|
||||
#undef VMULLO32
|
||||
#undef VSAD8
|
||||
#undef VSET1_8
|
||||
#undef VSET1_32
|
||||
#undef VSETZERO
|
||||
#undef VSLL32
|
||||
#undef VUNPACKLO8
|
||||
#undef VUNPACKHI8
|
||||
|
||||
#undef SUFFIX
|
||||
#undef ATTRIBUTES
|
||||
#undef VL
|
||||
#undef USE_VNNI
|
||||
#undef USE_AVX512
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
* x86/cpu_features.c - feature detection for x86 CPUs
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "../cpu_features_common.h" /* must be included first */
|
||||
#include "cpu_features.h"
|
||||
|
||||
#ifdef X86_CPU_FEATURES_KNOWN
|
||||
/* Runtime x86 CPU feature detection is supported. */
|
||||
|
||||
/* Execute the CPUID instruction. */
|
||||
static inline void
|
||||
cpuid(u32 leaf, u32 subleaf, u32 *a, u32 *b, u32 *c, u32 *d)
|
||||
{
|
||||
#ifdef _MSC_VER
|
||||
int result[4];
|
||||
|
||||
__cpuidex(result, leaf, subleaf);
|
||||
*a = result[0];
|
||||
*b = result[1];
|
||||
*c = result[2];
|
||||
*d = result[3];
|
||||
#else
|
||||
__asm__ volatile("cpuid" : "=a" (*a), "=b" (*b), "=c" (*c), "=d" (*d)
|
||||
: "a" (leaf), "c" (subleaf));
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Read an extended control register. */
|
||||
static inline u64
|
||||
read_xcr(u32 index)
|
||||
{
|
||||
#ifdef _MSC_VER
|
||||
return _xgetbv(index);
|
||||
#else
|
||||
u32 d, a;
|
||||
|
||||
/*
|
||||
* Execute the "xgetbv" instruction. Old versions of binutils do not
|
||||
* recognize this instruction, so list the raw bytes instead.
|
||||
*
|
||||
* This must be 'volatile' to prevent this code from being moved out
|
||||
* from under the check for OSXSAVE.
|
||||
*/
|
||||
__asm__ volatile(".byte 0x0f, 0x01, 0xd0" :
|
||||
"=d" (d), "=a" (a) : "c" (index));
|
||||
|
||||
return ((u64)d << 32) | a;
|
||||
#endif
|
||||
}
|
||||
|
||||
static const struct cpu_feature x86_cpu_feature_table[] = {
|
||||
{X86_CPU_FEATURE_SSE2, "sse2"},
|
||||
{X86_CPU_FEATURE_PCLMULQDQ, "pclmulqdq"},
|
||||
{X86_CPU_FEATURE_AVX, "avx"},
|
||||
{X86_CPU_FEATURE_AVX2, "avx2"},
|
||||
{X86_CPU_FEATURE_BMI2, "bmi2"},
|
||||
{X86_CPU_FEATURE_ZMM, "zmm"},
|
||||
{X86_CPU_FEATURE_AVX512BW, "avx512bw"},
|
||||
{X86_CPU_FEATURE_AVX512VL, "avx512vl"},
|
||||
{X86_CPU_FEATURE_VPCLMULQDQ, "vpclmulqdq"},
|
||||
{X86_CPU_FEATURE_AVX512VNNI, "avx512_vnni"},
|
||||
{X86_CPU_FEATURE_AVXVNNI, "avx_vnni"},
|
||||
};
|
||||
|
||||
volatile u32 libdeflate_x86_cpu_features = 0;
|
||||
|
||||
static inline bool
|
||||
os_supports_avx512(u64 xcr0)
|
||||
{
|
||||
#ifdef __APPLE__
|
||||
/*
|
||||
* The Darwin kernel had a bug where it could corrupt the opmask
|
||||
* registers. See
|
||||
* https://community.intel.com/t5/Software-Tuning-Performance/MacOS-Darwin-kernel-bug-clobbers-AVX-512-opmask-register-state/m-p/1327259
|
||||
* Darwin also does not initially set the XCR0 bits for AVX512, but they
|
||||
* are set if the thread tries to use AVX512 anyway. Thus, to safely
|
||||
* and consistently use AVX512 on macOS we'd need to check the kernel
|
||||
* version as well as detect AVX512 support using a macOS-specific
|
||||
* method. We don't bother with this, especially given Apple's
|
||||
* transition to arm64.
|
||||
*/
|
||||
return false;
|
||||
#else
|
||||
return (xcr0 & 0xe6) == 0xe6;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Don't use 512-bit vectors (ZMM registers) on Intel CPUs before Rocket Lake
|
||||
* and Sapphire Rapids, due to the overly-eager downclocking which can reduce
|
||||
* the performance of workloads that use ZMM registers only occasionally.
|
||||
*/
|
||||
static inline bool
|
||||
allow_512bit_vectors(const u32 manufacturer[3], u32 family, u32 model)
|
||||
{
|
||||
#ifdef TEST_SUPPORT__DO_NOT_USE
|
||||
return true;
|
||||
#endif
|
||||
if (memcmp(manufacturer, "GenuineIntel", 12) != 0)
|
||||
return true;
|
||||
if (family != 6)
|
||||
return true;
|
||||
switch (model) {
|
||||
case 85: /* Skylake (Server), Cascade Lake, Cooper Lake */
|
||||
case 106: /* Ice Lake (Server) */
|
||||
case 108: /* Ice Lake (Server) */
|
||||
case 126: /* Ice Lake (Client) */
|
||||
case 140: /* Tiger Lake */
|
||||
case 141: /* Tiger Lake */
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Initialize libdeflate_x86_cpu_features. */
|
||||
void libdeflate_init_x86_cpu_features(void)
|
||||
{
|
||||
u32 max_leaf;
|
||||
u32 manufacturer[3];
|
||||
u32 family, model;
|
||||
u32 a, b, c, d;
|
||||
u64 xcr0 = 0;
|
||||
u32 features = 0;
|
||||
|
||||
/* EAX=0: Highest Function Parameter and Manufacturer ID */
|
||||
cpuid(0, 0, &max_leaf, &manufacturer[0], &manufacturer[2],
|
||||
&manufacturer[1]);
|
||||
if (max_leaf < 1)
|
||||
goto out;
|
||||
|
||||
/* EAX=1: Processor Info and Feature Bits */
|
||||
cpuid(1, 0, &a, &b, &c, &d);
|
||||
family = (a >> 8) & 0xf;
|
||||
model = (a >> 4) & 0xf;
|
||||
if (family == 6 || family == 0xf)
|
||||
model += (a >> 12) & 0xf0;
|
||||
if (family == 0xf)
|
||||
family += (a >> 20) & 0xff;
|
||||
if (d & (1 << 26))
|
||||
features |= X86_CPU_FEATURE_SSE2;
|
||||
/*
|
||||
* No known CPUs have pclmulqdq without sse4.1, so in practice code
|
||||
* targeting pclmulqdq can use sse4.1 instructions. But to be safe,
|
||||
* explicitly check for both the pclmulqdq and sse4.1 bits.
|
||||
*/
|
||||
if ((c & (1 << 1)) && (c & (1 << 19)))
|
||||
features |= X86_CPU_FEATURE_PCLMULQDQ;
|
||||
if (c & (1 << 27))
|
||||
xcr0 = read_xcr(0);
|
||||
if ((c & (1 << 28)) && ((xcr0 & 0x6) == 0x6))
|
||||
features |= X86_CPU_FEATURE_AVX;
|
||||
|
||||
if (max_leaf < 7)
|
||||
goto out;
|
||||
|
||||
/* EAX=7, ECX=0: Extended Features */
|
||||
cpuid(7, 0, &a, &b, &c, &d);
|
||||
if (b & (1 << 8))
|
||||
features |= X86_CPU_FEATURE_BMI2;
|
||||
if ((xcr0 & 0x6) == 0x6) {
|
||||
if (b & (1 << 5))
|
||||
features |= X86_CPU_FEATURE_AVX2;
|
||||
if (c & (1 << 10))
|
||||
features |= X86_CPU_FEATURE_VPCLMULQDQ;
|
||||
}
|
||||
if (os_supports_avx512(xcr0)) {
|
||||
if (allow_512bit_vectors(manufacturer, family, model))
|
||||
features |= X86_CPU_FEATURE_ZMM;
|
||||
if (b & (1 << 30))
|
||||
features |= X86_CPU_FEATURE_AVX512BW;
|
||||
if (b & (1U << 31))
|
||||
features |= X86_CPU_FEATURE_AVX512VL;
|
||||
if (c & (1 << 11))
|
||||
features |= X86_CPU_FEATURE_AVX512VNNI;
|
||||
}
|
||||
|
||||
/* EAX=7, ECX=1: Extended Features */
|
||||
cpuid(7, 1, &a, &b, &c, &d);
|
||||
if ((a & (1 << 4)) && ((xcr0 & 0x6) == 0x6))
|
||||
features |= X86_CPU_FEATURE_AVXVNNI;
|
||||
|
||||
out:
|
||||
disable_cpu_features_for_testing(&features, x86_cpu_feature_table,
|
||||
ARRAY_LEN(x86_cpu_feature_table));
|
||||
|
||||
libdeflate_x86_cpu_features = features | X86_CPU_FEATURES_KNOWN;
|
||||
}
|
||||
|
||||
#endif /* X86_CPU_FEATURES_KNOWN */
|
||||
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
* x86/cpu_features.h - feature detection for x86 CPUs
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef LIB_X86_CPU_FEATURES_H
|
||||
#define LIB_X86_CPU_FEATURES_H
|
||||
|
||||
#include "../lib_common.h"
|
||||
|
||||
#if defined(ARCH_X86_32) || defined(ARCH_X86_64)
|
||||
|
||||
#define X86_CPU_FEATURE_SSE2 (1 << 0)
|
||||
#define X86_CPU_FEATURE_PCLMULQDQ (1 << 1)
|
||||
#define X86_CPU_FEATURE_AVX (1 << 2)
|
||||
#define X86_CPU_FEATURE_AVX2 (1 << 3)
|
||||
#define X86_CPU_FEATURE_BMI2 (1 << 4)
|
||||
/*
|
||||
* ZMM indicates whether 512-bit vectors (zmm registers) should be used. On
|
||||
* some CPUs, to avoid downclocking issues we don't set ZMM even if the CPU and
|
||||
* operating system support AVX-512. On these CPUs, we may still use AVX-512
|
||||
* instructions, but only with xmm and ymm registers.
|
||||
*/
|
||||
#define X86_CPU_FEATURE_ZMM (1 << 5)
|
||||
#define X86_CPU_FEATURE_AVX512BW (1 << 6)
|
||||
#define X86_CPU_FEATURE_AVX512VL (1 << 7)
|
||||
#define X86_CPU_FEATURE_VPCLMULQDQ (1 << 8)
|
||||
#define X86_CPU_FEATURE_AVX512VNNI (1 << 9)
|
||||
#define X86_CPU_FEATURE_AVXVNNI (1 << 10)
|
||||
|
||||
#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
|
||||
/* Runtime x86 CPU feature detection is supported. */
|
||||
# define X86_CPU_FEATURES_KNOWN (1U << 31)
|
||||
extern volatile u32 libdeflate_x86_cpu_features;
|
||||
|
||||
void libdeflate_init_x86_cpu_features(void);
|
||||
|
||||
static inline u32 get_x86_cpu_features(void)
|
||||
{
|
||||
if (libdeflate_x86_cpu_features == 0)
|
||||
libdeflate_init_x86_cpu_features();
|
||||
return libdeflate_x86_cpu_features;
|
||||
}
|
||||
/*
|
||||
* x86 intrinsics are also supported. Include the headers needed to use them.
|
||||
* Normally just immintrin.h suffices. With clang in MSVC compatibility mode,
|
||||
* immintrin.h incorrectly skips including sub-headers, so include those too.
|
||||
*/
|
||||
# include <immintrin.h>
|
||||
# if defined(_MSC_VER) && defined(__clang__)
|
||||
# include <tmmintrin.h>
|
||||
# include <smmintrin.h>
|
||||
# include <wmmintrin.h>
|
||||
# include <avxintrin.h>
|
||||
# include <avx2intrin.h>
|
||||
# include <avx512fintrin.h>
|
||||
# include <avx512bwintrin.h>
|
||||
# include <avx512vlintrin.h>
|
||||
# if __has_include(<avx512vlbwintrin.h>)
|
||||
# include <avx512vlbwintrin.h>
|
||||
# endif
|
||||
# if __has_include(<vpclmulqdqintrin.h>)
|
||||
# include <vpclmulqdqintrin.h>
|
||||
# endif
|
||||
# if __has_include(<avx512vnniintrin.h>)
|
||||
# include <avx512vnniintrin.h>
|
||||
# endif
|
||||
# if __has_include(<avx512vlvnniintrin.h>)
|
||||
# include <avx512vlvnniintrin.h>
|
||||
# endif
|
||||
# if __has_include(<avxvnniintrin.h>)
|
||||
# include <avxvnniintrin.h>
|
||||
# endif
|
||||
# endif
|
||||
#else
|
||||
static inline u32 get_x86_cpu_features(void) { return 0; }
|
||||
#endif
|
||||
|
||||
#if defined(__SSE2__) || \
|
||||
(defined(_MSC_VER) && \
|
||||
(defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
|
||||
# define HAVE_SSE2(features) 1
|
||||
# define HAVE_SSE2_NATIVE 1
|
||||
#else
|
||||
# define HAVE_SSE2(features) ((features) & X86_CPU_FEATURE_SSE2)
|
||||
# define HAVE_SSE2_NATIVE 0
|
||||
#endif
|
||||
|
||||
#if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
|
||||
(defined(_MSC_VER) && defined(__AVX2__))
|
||||
# define HAVE_PCLMULQDQ(features) 1
|
||||
#else
|
||||
# define HAVE_PCLMULQDQ(features) ((features) & X86_CPU_FEATURE_PCLMULQDQ)
|
||||
#endif
|
||||
|
||||
#ifdef __AVX__
|
||||
# define HAVE_AVX(features) 1
|
||||
#else
|
||||
# define HAVE_AVX(features) ((features) & X86_CPU_FEATURE_AVX)
|
||||
#endif
|
||||
|
||||
#ifdef __AVX2__
|
||||
# define HAVE_AVX2(features) 1
|
||||
#else
|
||||
# define HAVE_AVX2(features) ((features) & X86_CPU_FEATURE_AVX2)
|
||||
#endif
|
||||
|
||||
#if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
|
||||
# define HAVE_BMI2(features) 1
|
||||
# define HAVE_BMI2_NATIVE 1
|
||||
#else
|
||||
# define HAVE_BMI2(features) ((features) & X86_CPU_FEATURE_BMI2)
|
||||
# define HAVE_BMI2_NATIVE 0
|
||||
#endif
|
||||
|
||||
#ifdef __AVX512BW__
|
||||
# define HAVE_AVX512BW(features) 1
|
||||
#else
|
||||
# define HAVE_AVX512BW(features) ((features) & X86_CPU_FEATURE_AVX512BW)
|
||||
#endif
|
||||
|
||||
#ifdef __AVX512VL__
|
||||
# define HAVE_AVX512VL(features) 1
|
||||
#else
|
||||
# define HAVE_AVX512VL(features) ((features) & X86_CPU_FEATURE_AVX512VL)
|
||||
#endif
|
||||
|
||||
#ifdef __VPCLMULQDQ__
|
||||
# define HAVE_VPCLMULQDQ(features) 1
|
||||
#else
|
||||
# define HAVE_VPCLMULQDQ(features) ((features) & X86_CPU_FEATURE_VPCLMULQDQ)
|
||||
#endif
|
||||
|
||||
#ifdef __AVX512VNNI__
|
||||
# define HAVE_AVX512VNNI(features) 1
|
||||
#else
|
||||
# define HAVE_AVX512VNNI(features) ((features) & X86_CPU_FEATURE_AVX512VNNI)
|
||||
#endif
|
||||
|
||||
#ifdef __AVXVNNI__
|
||||
# define HAVE_AVXVNNI(features) 1
|
||||
#else
|
||||
# define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
|
||||
#endif
|
||||
|
||||
#endif /* ARCH_X86_32 || ARCH_X86_64 */
|
||||
|
||||
#endif /* LIB_X86_CPU_FEATURES_H */
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
* x86/crc32_impl.h - x86 implementations of the gzip CRC-32 algorithm
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef LIB_X86_CRC32_IMPL_H
|
||||
#define LIB_X86_CRC32_IMPL_H
|
||||
|
||||
#include "cpu_features.h"
|
||||
|
||||
/*
|
||||
* pshufb(x, shift_tab[len..len+15]) left shifts x by 16-len bytes.
|
||||
* pshufb(x, shift_tab[len+16..len+31]) right shifts x by len bytes.
|
||||
*/
|
||||
static const u8 MAYBE_UNUSED shift_tab[48] = {
|
||||
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
|
||||
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
|
||||
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
||||
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
|
||||
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
|
||||
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
|
||||
};
|
||||
|
||||
#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
|
||||
/*
|
||||
* PCLMULQDQ implementation. This targets PCLMULQDQ+SSE4.1, since in practice
|
||||
* all CPUs that support PCLMULQDQ also support SSE4.1.
|
||||
*/
|
||||
# define crc32_x86_pclmulqdq crc32_x86_pclmulqdq
|
||||
# define SUFFIX _pclmulqdq
|
||||
# define ATTRIBUTES _target_attribute("pclmul,sse4.1")
|
||||
# define VL 16
|
||||
# define USE_AVX512 0
|
||||
# include "crc32_pclmul_template.h"
|
||||
|
||||
/*
|
||||
* PCLMULQDQ/AVX implementation. Same as above, but this is compiled with AVX
|
||||
* enabled so that the compiler can generate VEX-coded instructions which can be
|
||||
* slightly more efficient. It still uses 128-bit vectors.
|
||||
*/
|
||||
# define crc32_x86_pclmulqdq_avx crc32_x86_pclmulqdq_avx
|
||||
# define SUFFIX _pclmulqdq_avx
|
||||
# define ATTRIBUTES _target_attribute("pclmul,avx")
|
||||
# define VL 16
|
||||
# define USE_AVX512 0
|
||||
# include "crc32_pclmul_template.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* VPCLMULQDQ/AVX2 implementation. This is used on CPUs that have AVX2 and
|
||||
* VPCLMULQDQ but don't have AVX-512, for example Intel Alder Lake.
|
||||
*
|
||||
* Currently this can't be enabled with MSVC because MSVC has a bug where it
|
||||
* incorrectly assumes that VPCLMULQDQ implies AVX-512:
|
||||
* https://developercommunity.visualstudio.com/t/Compiler-incorrectly-assumes-VAES-and-VP/10578785
|
||||
*
|
||||
* gcc 8.1 and 8.2 had a similar bug where they assumed that
|
||||
* _mm256_clmulepi64_epi128() always needed AVX512. It's fixed in gcc 8.3.
|
||||
*
|
||||
* _mm256_zextsi128_si256() requires gcc 10.
|
||||
*/
|
||||
#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000)) && \
|
||||
!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
|
||||
# define crc32_x86_vpclmulqdq_avx2 crc32_x86_vpclmulqdq_avx2
|
||||
# define SUFFIX _vpclmulqdq_avx2
|
||||
# define ATTRIBUTES _target_attribute("vpclmulqdq,pclmul,avx2")
|
||||
# define VL 32
|
||||
# define USE_AVX512 0
|
||||
# include "crc32_pclmul_template.h"
|
||||
#endif
|
||||
|
||||
#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
|
||||
!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
|
||||
/*
|
||||
* VPCLMULQDQ/AVX512 implementation using 256-bit vectors. This is very similar
|
||||
* to the VPCLMULQDQ/AVX2 implementation but takes advantage of the vpternlog
|
||||
* instruction and more registers. This is used on certain older Intel CPUs,
|
||||
* specifically Ice Lake and Tiger Lake, which support VPCLMULQDQ and AVX512 but
|
||||
* downclock a bit too eagerly when ZMM registers are used.
|
||||
*
|
||||
* _mm256_zextsi128_si256() requires gcc 10.
|
||||
*/
|
||||
# define crc32_x86_vpclmulqdq_avx512_vl256 crc32_x86_vpclmulqdq_avx512_vl256
|
||||
# define SUFFIX _vpclmulqdq_avx512_vl256
|
||||
# define ATTRIBUTES _target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
|
||||
# define VL 32
|
||||
# define USE_AVX512 1
|
||||
# include "crc32_pclmul_template.h"
|
||||
|
||||
/*
|
||||
* VPCLMULQDQ/AVX512 implementation using 512-bit vectors. This is used on CPUs
|
||||
* that have a good AVX-512 implementation including VPCLMULQDQ.
|
||||
*
|
||||
* _mm512_zextsi128_si512() requires gcc 10.
|
||||
*/
|
||||
# define crc32_x86_vpclmulqdq_avx512_vl512 crc32_x86_vpclmulqdq_avx512_vl512
|
||||
# define SUFFIX _vpclmulqdq_avx512_vl512
|
||||
# define ATTRIBUTES _target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
|
||||
# define VL 64
|
||||
# define USE_AVX512 1
|
||||
# include "crc32_pclmul_template.h"
|
||||
#endif
|
||||
|
||||
static inline crc32_func_t
|
||||
arch_select_crc32_func(void)
|
||||
{
|
||||
const u32 features MAYBE_UNUSED = get_x86_cpu_features();
|
||||
|
||||
#ifdef crc32_x86_vpclmulqdq_avx512_vl512
|
||||
if ((features & X86_CPU_FEATURE_ZMM) &&
|
||||
HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
|
||||
HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
|
||||
return crc32_x86_vpclmulqdq_avx512_vl512;
|
||||
#endif
|
||||
#ifdef crc32_x86_vpclmulqdq_avx512_vl256
|
||||
if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
|
||||
HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
|
||||
return crc32_x86_vpclmulqdq_avx512_vl256;
|
||||
#endif
|
||||
#ifdef crc32_x86_vpclmulqdq_avx2
|
||||
if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
|
||||
HAVE_AVX2(features))
|
||||
return crc32_x86_vpclmulqdq_avx2;
|
||||
#endif
|
||||
#ifdef crc32_x86_pclmulqdq_avx
|
||||
if (HAVE_PCLMULQDQ(features) && HAVE_AVX(features))
|
||||
return crc32_x86_pclmulqdq_avx;
|
||||
#endif
|
||||
#ifdef crc32_x86_pclmulqdq
|
||||
if (HAVE_PCLMULQDQ(features))
|
||||
return crc32_x86_pclmulqdq;
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
#define arch_select_crc32_func arch_select_crc32_func
|
||||
|
||||
#endif /* LIB_X86_CRC32_IMPL_H */
|
||||
@@ -0,0 +1,424 @@
|
||||
/*
|
||||
* x86/crc32_pclmul_template.h - gzip CRC-32 with PCLMULQDQ instructions
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is a "template" for instantiating PCLMULQDQ-based crc32_x86
|
||||
* functions. The "parameters" are:
|
||||
*
|
||||
* SUFFIX:
|
||||
* Name suffix to append to all instantiated functions.
|
||||
* ATTRIBUTES:
|
||||
* Target function attributes to use. Must satisfy the dependencies of the
|
||||
* other parameters as follows:
|
||||
* VL=16 && USE_AVX512=0: at least pclmul,sse4.1
|
||||
* VL=32 && USE_AVX512=0: at least vpclmulqdq,pclmul,avx2
|
||||
* VL=32 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
|
||||
* VL=64 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
|
||||
* (Other combinations are not useful and have not been tested.)
|
||||
* VL:
|
||||
* Vector length in bytes. Must be 16, 32, or 64.
|
||||
* USE_AVX512:
|
||||
* If 1, take advantage of AVX-512 features such as masking and the
|
||||
* vpternlog instruction. This doesn't enable the use of 512-bit vectors;
|
||||
* the vector length is controlled by VL. If 0, assume that the CPU might
|
||||
* not support AVX-512.
|
||||
*
|
||||
* The overall algorithm used is CRC folding with carryless multiplication
|
||||
* instructions. Note that the x86 crc32 instruction cannot be used, as it is
|
||||
* for a different polynomial, not the gzip one. For an explanation of CRC
|
||||
* folding with carryless multiplication instructions, see
|
||||
* scripts/gen-crc32-consts.py and the following blog posts and papers:
|
||||
*
|
||||
* "An alternative exposition of crc32_4k_pclmulqdq"
|
||||
* https://www.corsix.org/content/alternative-exposition-crc32_4k_pclmulqdq
|
||||
*
|
||||
* "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
|
||||
* https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
|
||||
*
|
||||
* The original pclmulqdq instruction does one 64x64 to 128-bit carryless
|
||||
* multiplication. The VPCLMULQDQ feature added instructions that do two
|
||||
* parallel 64x64 to 128-bit carryless multiplications in combination with AVX
|
||||
* or AVX512VL, or four in combination with AVX512F.
|
||||
*/
|
||||
|
||||
#if VL == 16
|
||||
# define vec_t __m128i
|
||||
# define fold_vec fold_vec128
|
||||
# define VLOADU(p) _mm_loadu_si128((const void *)(p))
|
||||
# define VXOR(a, b) _mm_xor_si128((a), (b))
|
||||
# define M128I_TO_VEC(a) a
|
||||
# define MULTS_8V _mm_set_epi64x(CRC32_X991_MODG, CRC32_X1055_MODG)
|
||||
# define MULTS_4V _mm_set_epi64x(CRC32_X479_MODG, CRC32_X543_MODG)
|
||||
# define MULTS_2V _mm_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG)
|
||||
# define MULTS_1V _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG)
|
||||
#elif VL == 32
|
||||
# define vec_t __m256i
|
||||
# define fold_vec fold_vec256
|
||||
# define VLOADU(p) _mm256_loadu_si256((const void *)(p))
|
||||
# define VXOR(a, b) _mm256_xor_si256((a), (b))
|
||||
# define M128I_TO_VEC(a) _mm256_zextsi128_si256(a)
|
||||
# define MULTS(a, b) _mm256_set_epi64x(a, b, a, b)
|
||||
# define MULTS_8V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
|
||||
# define MULTS_4V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
|
||||
# define MULTS_2V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
|
||||
# define MULTS_1V MULTS(CRC32_X223_MODG, CRC32_X287_MODG)
|
||||
#elif VL == 64
|
||||
# define vec_t __m512i
|
||||
# define fold_vec fold_vec512
|
||||
# define VLOADU(p) _mm512_loadu_si512((const void *)(p))
|
||||
# define VXOR(a, b) _mm512_xor_si512((a), (b))
|
||||
# define M128I_TO_VEC(a) _mm512_zextsi128_si512(a)
|
||||
# define MULTS(a, b) _mm512_set_epi64(a, b, a, b, a, b, a, b)
|
||||
# define MULTS_8V MULTS(CRC32_X4063_MODG, CRC32_X4127_MODG)
|
||||
# define MULTS_4V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
|
||||
# define MULTS_2V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
|
||||
# define MULTS_1V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
|
||||
#else
|
||||
# error "unsupported vector length"
|
||||
#endif
|
||||
|
||||
#undef fold_vec128
|
||||
static forceinline ATTRIBUTES __m128i
|
||||
ADD_SUFFIX(fold_vec128)(__m128i src, __m128i dst, __m128i /* __v2du */ mults)
|
||||
{
|
||||
dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x00));
|
||||
dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x11));
|
||||
return dst;
|
||||
}
|
||||
#define fold_vec128 ADD_SUFFIX(fold_vec128)
|
||||
|
||||
#if VL >= 32
|
||||
#undef fold_vec256
|
||||
static forceinline ATTRIBUTES __m256i
|
||||
ADD_SUFFIX(fold_vec256)(__m256i src, __m256i dst, __m256i /* __v4du */ mults)
|
||||
{
|
||||
#if USE_AVX512
|
||||
/* vpternlog with immediate 0x96 is a three-argument XOR. */
|
||||
return _mm256_ternarylogic_epi32(
|
||||
_mm256_clmulepi64_epi128(src, mults, 0x00),
|
||||
_mm256_clmulepi64_epi128(src, mults, 0x11),
|
||||
dst,
|
||||
0x96);
|
||||
#else
|
||||
return _mm256_xor_si256(
|
||||
_mm256_xor_si256(dst,
|
||||
_mm256_clmulepi64_epi128(src, mults, 0x00)),
|
||||
_mm256_clmulepi64_epi128(src, mults, 0x11));
|
||||
#endif
|
||||
}
|
||||
#define fold_vec256 ADD_SUFFIX(fold_vec256)
|
||||
#endif /* VL >= 32 */
|
||||
|
||||
#if VL >= 64
|
||||
#undef fold_vec512
|
||||
static forceinline ATTRIBUTES __m512i
|
||||
ADD_SUFFIX(fold_vec512)(__m512i src, __m512i dst, __m512i /* __v8du */ mults)
|
||||
{
|
||||
/* vpternlog with immediate 0x96 is a three-argument XOR. */
|
||||
return _mm512_ternarylogic_epi32(
|
||||
_mm512_clmulepi64_epi128(src, mults, 0x00),
|
||||
_mm512_clmulepi64_epi128(src, mults, 0x11),
|
||||
dst,
|
||||
0x96);
|
||||
}
|
||||
#define fold_vec512 ADD_SUFFIX(fold_vec512)
|
||||
#endif /* VL >= 64 */
|
||||
|
||||
/*
|
||||
* Given 'x' containing a 16-byte polynomial, and a pointer 'p' that points to
|
||||
* the next '1 <= len <= 15' data bytes, rearrange the concatenation of 'x' and
|
||||
* the data into vectors x0 and x1 that contain 'len' bytes and 16 bytes,
|
||||
* respectively. Then fold x0 into x1 and return the result.
|
||||
* Assumes that 'p + len - 16' is in-bounds.
|
||||
*/
|
||||
#undef fold_lessthan16bytes
|
||||
static forceinline ATTRIBUTES __m128i
|
||||
ADD_SUFFIX(fold_lessthan16bytes)(__m128i x, const u8 *p, size_t len,
|
||||
__m128i /* __v2du */ mults_128b)
|
||||
{
|
||||
__m128i lshift = _mm_loadu_si128((const void *)&shift_tab[len]);
|
||||
__m128i rshift = _mm_loadu_si128((const void *)&shift_tab[len + 16]);
|
||||
__m128i x0, x1;
|
||||
|
||||
/* x0 = x left-shifted by '16 - len' bytes */
|
||||
x0 = _mm_shuffle_epi8(x, lshift);
|
||||
|
||||
/*
|
||||
* x1 = the last '16 - len' bytes from x (i.e. x right-shifted by 'len'
|
||||
* bytes) followed by the remaining data.
|
||||
*/
|
||||
x1 = _mm_blendv_epi8(_mm_shuffle_epi8(x, rshift),
|
||||
_mm_loadu_si128((const void *)(p + len - 16)),
|
||||
/* msb 0/1 of each byte selects byte from arg1/2 */
|
||||
rshift);
|
||||
|
||||
return fold_vec128(x0, x1, mults_128b);
|
||||
}
|
||||
#define fold_lessthan16bytes ADD_SUFFIX(fold_lessthan16bytes)
|
||||
|
||||
static ATTRIBUTES u32
|
||||
ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
|
||||
{
|
||||
/*
|
||||
* mults_{N}v are the vectors of multipliers for folding across N vec_t
|
||||
* vectors, i.e. N*VL*8 bits. mults_128b are the two multipliers for
|
||||
* folding across 128 bits. mults_128b differs from mults_1v when
|
||||
* VL != 16. All multipliers are 64-bit, to match what pclmulqdq needs,
|
||||
* but since this is for CRC-32 only their low 32 bits are nonzero.
|
||||
* For more details, see scripts/gen-crc32-consts.py.
|
||||
*/
|
||||
const vec_t mults_8v = MULTS_8V;
|
||||
const vec_t mults_4v = MULTS_4V;
|
||||
const vec_t mults_2v = MULTS_2V;
|
||||
const vec_t mults_1v = MULTS_1V;
|
||||
const __m128i mults_128b = _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG);
|
||||
const __m128i barrett_reduction_constants =
|
||||
_mm_set_epi64x(CRC32_BARRETT_CONSTANT_2, CRC32_BARRETT_CONSTANT_1);
|
||||
vec_t v0, v1, v2, v3, v4, v5, v6, v7;
|
||||
__m128i x0 = _mm_cvtsi32_si128(crc);
|
||||
__m128i x1;
|
||||
|
||||
if (len < 8*VL) {
|
||||
if (len < VL) {
|
||||
STATIC_ASSERT(VL == 16 || VL == 32 || VL == 64);
|
||||
if (len < 16) {
|
||||
#if USE_AVX512
|
||||
if (len < 4)
|
||||
return crc32_slice1(crc, p, len);
|
||||
/*
|
||||
* Handle 4 <= len <= 15 bytes by doing a masked
|
||||
* load, XOR'ing the current CRC with the first
|
||||
* 4 bytes, left-shifting by '16 - len' bytes to
|
||||
* align the result to the end of x0 (so that it
|
||||
* becomes the low-order coefficients of a
|
||||
* 128-bit polynomial), and then doing the usual
|
||||
* reduction from 128 bits to 32 bits.
|
||||
*/
|
||||
x0 = _mm_xor_si128(
|
||||
x0, _mm_maskz_loadu_epi8((1 << len) - 1, p));
|
||||
x0 = _mm_shuffle_epi8(
|
||||
x0, _mm_loadu_si128((const void *)&shift_tab[len]));
|
||||
goto reduce_x0;
|
||||
#else
|
||||
return crc32_slice1(crc, p, len);
|
||||
#endif
|
||||
}
|
||||
/*
|
||||
* Handle 16 <= len < VL bytes where VL is 32 or 64.
|
||||
* Use 128-bit instructions so that these lengths aren't
|
||||
* slower with VL > 16 than with VL=16.
|
||||
*/
|
||||
x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
|
||||
if (len >= 32) {
|
||||
x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 16)),
|
||||
mults_128b);
|
||||
if (len >= 48)
|
||||
x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 32)),
|
||||
mults_128b);
|
||||
}
|
||||
p += len & ~15;
|
||||
goto less_than_16_remaining;
|
||||
}
|
||||
v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
|
||||
if (len < 2*VL) {
|
||||
p += VL;
|
||||
goto less_than_vl_remaining;
|
||||
}
|
||||
v1 = VLOADU(p + 1*VL);
|
||||
if (len < 4*VL) {
|
||||
p += 2*VL;
|
||||
goto less_than_2vl_remaining;
|
||||
}
|
||||
v2 = VLOADU(p + 2*VL);
|
||||
v3 = VLOADU(p + 3*VL);
|
||||
p += 4*VL;
|
||||
} else {
|
||||
/*
|
||||
* If the length is large and the pointer is misaligned, align
|
||||
* it. For smaller lengths, just take the misaligned load
|
||||
* penalty. Note that on recent x86 CPUs, vmovdqu with an
|
||||
* aligned address is just as fast as vmovdqa, so there's no
|
||||
* need to use vmovdqa in the main loop.
|
||||
*/
|
||||
if (len > 65536 && ((uintptr_t)p & (VL-1))) {
|
||||
size_t align = -(uintptr_t)p & (VL-1);
|
||||
|
||||
len -= align;
|
||||
x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
|
||||
p += 16;
|
||||
if (align & 15) {
|
||||
x0 = fold_lessthan16bytes(x0, p, align & 15,
|
||||
mults_128b);
|
||||
p += align & 15;
|
||||
align &= ~15;
|
||||
}
|
||||
while (align) {
|
||||
x0 = fold_vec128(x0, *(const __m128i *)p,
|
||||
mults_128b);
|
||||
p += 16;
|
||||
align -= 16;
|
||||
}
|
||||
v0 = M128I_TO_VEC(x0);
|
||||
# if VL == 32
|
||||
v0 = _mm256_inserti128_si256(v0, *(const __m128i *)p, 1);
|
||||
# elif VL == 64
|
||||
v0 = _mm512_inserti32x4(v0, *(const __m128i *)p, 1);
|
||||
v0 = _mm512_inserti64x4(v0, *(const __m256i *)(p + 16), 1);
|
||||
# endif
|
||||
p -= 16;
|
||||
} else {
|
||||
v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
|
||||
}
|
||||
v1 = VLOADU(p + 1*VL);
|
||||
v2 = VLOADU(p + 2*VL);
|
||||
v3 = VLOADU(p + 3*VL);
|
||||
v4 = VLOADU(p + 4*VL);
|
||||
v5 = VLOADU(p + 5*VL);
|
||||
v6 = VLOADU(p + 6*VL);
|
||||
v7 = VLOADU(p + 7*VL);
|
||||
p += 8*VL;
|
||||
|
||||
/*
|
||||
* This is the main loop, processing 8*VL bytes per iteration.
|
||||
* 4*VL is usually enough and would result in smaller code, but
|
||||
* Skylake and Cascade Lake need 8*VL to get full performance.
|
||||
*/
|
||||
while (len >= 16*VL) {
|
||||
v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_8v);
|
||||
v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_8v);
|
||||
v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_8v);
|
||||
v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_8v);
|
||||
v4 = fold_vec(v4, VLOADU(p + 4*VL), mults_8v);
|
||||
v5 = fold_vec(v5, VLOADU(p + 5*VL), mults_8v);
|
||||
v6 = fold_vec(v6, VLOADU(p + 6*VL), mults_8v);
|
||||
v7 = fold_vec(v7, VLOADU(p + 7*VL), mults_8v);
|
||||
p += 8*VL;
|
||||
len -= 8*VL;
|
||||
}
|
||||
|
||||
/* Fewer than 8*VL bytes remain. */
|
||||
v0 = fold_vec(v0, v4, mults_4v);
|
||||
v1 = fold_vec(v1, v5, mults_4v);
|
||||
v2 = fold_vec(v2, v6, mults_4v);
|
||||
v3 = fold_vec(v3, v7, mults_4v);
|
||||
if (len & (4*VL)) {
|
||||
v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_4v);
|
||||
v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_4v);
|
||||
v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_4v);
|
||||
v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_4v);
|
||||
p += 4*VL;
|
||||
}
|
||||
}
|
||||
/* Fewer than 4*VL bytes remain. */
|
||||
v0 = fold_vec(v0, v2, mults_2v);
|
||||
v1 = fold_vec(v1, v3, mults_2v);
|
||||
if (len & (2*VL)) {
|
||||
v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_2v);
|
||||
v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_2v);
|
||||
p += 2*VL;
|
||||
}
|
||||
less_than_2vl_remaining:
|
||||
/* Fewer than 2*VL bytes remain. */
|
||||
v0 = fold_vec(v0, v1, mults_1v);
|
||||
if (len & VL) {
|
||||
v0 = fold_vec(v0, VLOADU(p), mults_1v);
|
||||
p += VL;
|
||||
}
|
||||
less_than_vl_remaining:
|
||||
/*
|
||||
* Fewer than VL bytes remain. Reduce v0 (length VL bytes) to x0
|
||||
* (length 16 bytes) and fold in any 16-byte data segments that remain.
|
||||
*/
|
||||
#if VL == 16
|
||||
x0 = v0;
|
||||
#else
|
||||
{
|
||||
#if VL == 32
|
||||
__m256i y0 = v0;
|
||||
#else
|
||||
const __m256i mults_256b =
|
||||
_mm256_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG,
|
||||
CRC32_X223_MODG, CRC32_X287_MODG);
|
||||
__m256i y0 = fold_vec256(_mm512_extracti64x4_epi64(v0, 0),
|
||||
_mm512_extracti64x4_epi64(v0, 1),
|
||||
mults_256b);
|
||||
if (len & 32) {
|
||||
y0 = fold_vec256(y0, _mm256_loadu_si256((const void *)p),
|
||||
mults_256b);
|
||||
p += 32;
|
||||
}
|
||||
#endif
|
||||
x0 = fold_vec128(_mm256_extracti128_si256(y0, 0),
|
||||
_mm256_extracti128_si256(y0, 1), mults_128b);
|
||||
}
|
||||
if (len & 16) {
|
||||
x0 = fold_vec128(x0, _mm_loadu_si128((const void *)p),
|
||||
mults_128b);
|
||||
p += 16;
|
||||
}
|
||||
#endif
|
||||
less_than_16_remaining:
|
||||
len &= 15;
|
||||
|
||||
/* Handle any remainder of 1 to 15 bytes. */
|
||||
if (len)
|
||||
x0 = fold_lessthan16bytes(x0, p, len, mults_128b);
|
||||
#if USE_AVX512
|
||||
reduce_x0:
|
||||
#endif
|
||||
/*
|
||||
* Multiply the remaining 128-bit message polynomial 'x0' by x^32, then
|
||||
* reduce it modulo the generator polynomial G. This gives the CRC.
|
||||
*
|
||||
* This implementation matches that used in crc-pclmul-template.S from
|
||||
* https://lore.kernel.org/r/20250210174540.161705-4-ebiggers@kernel.org/
|
||||
* with the parameters n=32 and LSB_CRC=1 (what the gzip CRC uses). See
|
||||
* there for a detailed explanation of the math used here.
|
||||
*/
|
||||
x0 = _mm_xor_si128(_mm_clmulepi64_si128(x0, mults_128b, 0x10),
|
||||
_mm_bsrli_si128(x0, 8));
|
||||
x1 = _mm_clmulepi64_si128(x0, barrett_reduction_constants, 0x00);
|
||||
x1 = _mm_clmulepi64_si128(x1, barrett_reduction_constants, 0x10);
|
||||
x0 = _mm_xor_si128(x0, x1);
|
||||
return _mm_extract_epi32(x0, 2);
|
||||
}
|
||||
|
||||
#undef vec_t
|
||||
#undef fold_vec
|
||||
#undef VLOADU
|
||||
#undef VXOR
|
||||
#undef M128I_TO_VEC
|
||||
#undef MULTS
|
||||
#undef MULTS_8V
|
||||
#undef MULTS_4V
|
||||
#undef MULTS_2V
|
||||
#undef MULTS_1V
|
||||
|
||||
#undef SUFFIX
|
||||
#undef ATTRIBUTES
|
||||
#undef VL
|
||||
#undef USE_AVX512
|
||||
@@ -0,0 +1,57 @@
|
||||
#ifndef LIB_X86_DECOMPRESS_IMPL_H
|
||||
#define LIB_X86_DECOMPRESS_IMPL_H
|
||||
|
||||
#include "cpu_features.h"
|
||||
|
||||
/*
|
||||
* BMI2 optimized decompression function.
|
||||
*
|
||||
* With gcc and clang we just compile the whole function with
|
||||
* __attribute__((target("bmi2"))), and the compiler uses bmi2 automatically.
|
||||
*
|
||||
* With MSVC, there is no target function attribute, but it's still possible to
|
||||
* use bmi2 intrinsics explicitly. Currently we mostly don't, but there's a
|
||||
* case in which we do (see below), so we at least take advantage of that.
|
||||
* However, MSVC from VS2017 (toolset v141) apparently miscompiles the _bzhi_*()
|
||||
* intrinsics. It seems to be fixed in VS2022. Hence, use MSVC_PREREQ(1930).
|
||||
*/
|
||||
#if defined(__GNUC__) || defined(__clang__) || MSVC_PREREQ(1930)
|
||||
# define deflate_decompress_bmi2 deflate_decompress_bmi2
|
||||
# define FUNCNAME deflate_decompress_bmi2
|
||||
# define ATTRIBUTES _target_attribute("bmi2")
|
||||
/*
|
||||
* Even with __attribute__((target("bmi2"))), gcc doesn't reliably use the
|
||||
* bzhi instruction for 'word & BITMASK(count)'. So use the bzhi intrinsic
|
||||
* explicitly. EXTRACT_VARBITS() is equivalent to 'word & BITMASK(count)';
|
||||
* EXTRACT_VARBITS8() is equivalent to 'word & BITMASK((u8)count)'.
|
||||
* Nevertheless, their implementation using the bzhi intrinsic is identical,
|
||||
* as the bzhi instruction truncates the count to 8 bits implicitly.
|
||||
*/
|
||||
# ifndef __clang__
|
||||
# ifdef ARCH_X86_64
|
||||
# define EXTRACT_VARBITS(word, count) _bzhi_u64((word), (count))
|
||||
# define EXTRACT_VARBITS8(word, count) _bzhi_u64((word), (count))
|
||||
# else
|
||||
# define EXTRACT_VARBITS(word, count) _bzhi_u32((word), (count))
|
||||
# define EXTRACT_VARBITS8(word, count) _bzhi_u32((word), (count))
|
||||
# endif
|
||||
# endif
|
||||
# include "../decompress_template.h"
|
||||
#endif
|
||||
|
||||
#if defined(deflate_decompress_bmi2) && HAVE_BMI2_NATIVE
|
||||
#define DEFAULT_IMPL deflate_decompress_bmi2
|
||||
#else
|
||||
static inline decompress_func_t
|
||||
arch_select_decompress_func(void)
|
||||
{
|
||||
#ifdef deflate_decompress_bmi2
|
||||
if (HAVE_BMI2(get_x86_cpu_features()))
|
||||
return deflate_decompress_bmi2;
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
#define arch_select_decompress_func arch_select_decompress_func
|
||||
#endif
|
||||
|
||||
#endif /* LIB_X86_DECOMPRESS_IMPL_H */
|
||||
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* x86/matchfinder_impl.h - x86 implementations of matchfinder functions
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef LIB_X86_MATCHFINDER_IMPL_H
|
||||
#define LIB_X86_MATCHFINDER_IMPL_H
|
||||
|
||||
#include "cpu_features.h"
|
||||
|
||||
#ifdef __AVX2__
|
||||
static forceinline void
|
||||
matchfinder_init_avx2(mf_pos_t *data, size_t size)
|
||||
{
|
||||
__m256i *p = (__m256i *)data;
|
||||
__m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
|
||||
|
||||
STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
|
||||
STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
|
||||
STATIC_ASSERT(sizeof(mf_pos_t) == 2);
|
||||
|
||||
do {
|
||||
p[0] = v;
|
||||
p[1] = v;
|
||||
p[2] = v;
|
||||
p[3] = v;
|
||||
p += 4;
|
||||
size -= 4 * sizeof(*p);
|
||||
} while (size != 0);
|
||||
}
|
||||
#define matchfinder_init matchfinder_init_avx2
|
||||
|
||||
static forceinline void
|
||||
matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
|
||||
{
|
||||
__m256i *p = (__m256i *)data;
|
||||
__m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
|
||||
|
||||
STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
|
||||
STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
|
||||
STATIC_ASSERT(sizeof(mf_pos_t) == 2);
|
||||
|
||||
do {
|
||||
/* PADDSW: Add Packed Signed Integers With Signed Saturation */
|
||||
p[0] = _mm256_adds_epi16(p[0], v);
|
||||
p[1] = _mm256_adds_epi16(p[1], v);
|
||||
p[2] = _mm256_adds_epi16(p[2], v);
|
||||
p[3] = _mm256_adds_epi16(p[3], v);
|
||||
p += 4;
|
||||
size -= 4 * sizeof(*p);
|
||||
} while (size != 0);
|
||||
}
|
||||
#define matchfinder_rebase matchfinder_rebase_avx2
|
||||
|
||||
#elif HAVE_SSE2_NATIVE
|
||||
static forceinline void
|
||||
matchfinder_init_sse2(mf_pos_t *data, size_t size)
|
||||
{
|
||||
__m128i *p = (__m128i *)data;
|
||||
__m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL);
|
||||
|
||||
STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
|
||||
STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
|
||||
STATIC_ASSERT(sizeof(mf_pos_t) == 2);
|
||||
|
||||
do {
|
||||
p[0] = v;
|
||||
p[1] = v;
|
||||
p[2] = v;
|
||||
p[3] = v;
|
||||
p += 4;
|
||||
size -= 4 * sizeof(*p);
|
||||
} while (size != 0);
|
||||
}
|
||||
#define matchfinder_init matchfinder_init_sse2
|
||||
|
||||
static forceinline void
|
||||
matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
|
||||
{
|
||||
__m128i *p = (__m128i *)data;
|
||||
__m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
|
||||
|
||||
STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
|
||||
STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
|
||||
STATIC_ASSERT(sizeof(mf_pos_t) == 2);
|
||||
|
||||
do {
|
||||
/* PADDSW: Add Packed Signed Integers With Signed Saturation */
|
||||
p[0] = _mm_adds_epi16(p[0], v);
|
||||
p[1] = _mm_adds_epi16(p[1], v);
|
||||
p[2] = _mm_adds_epi16(p[2], v);
|
||||
p[3] = _mm_adds_epi16(p[3], v);
|
||||
p += 4;
|
||||
size -= 4 * sizeof(*p);
|
||||
} while (size != 0);
|
||||
}
|
||||
#define matchfinder_rebase matchfinder_rebase_sse2
|
||||
#endif /* HAVE_SSE2_NATIVE */
|
||||
|
||||
#endif /* LIB_X86_MATCHFINDER_IMPL_H */
|
||||
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
* zlib_constants.h - constants for the zlib wrapper format
|
||||
*/
|
||||
|
||||
#ifndef LIB_ZLIB_CONSTANTS_H
|
||||
#define LIB_ZLIB_CONSTANTS_H
|
||||
|
||||
#define ZLIB_MIN_HEADER_SIZE 2
|
||||
#define ZLIB_FOOTER_SIZE 4
|
||||
#define ZLIB_MIN_OVERHEAD (ZLIB_MIN_HEADER_SIZE + ZLIB_FOOTER_SIZE)
|
||||
|
||||
#define ZLIB_CM_DEFLATE 8
|
||||
|
||||
#define ZLIB_CINFO_32K_WINDOW 7
|
||||
|
||||
#define ZLIB_FASTEST_COMPRESSION 0
|
||||
#define ZLIB_FAST_COMPRESSION 1
|
||||
#define ZLIB_DEFAULT_COMPRESSION 2
|
||||
#define ZLIB_SLOWEST_COMPRESSION 3
|
||||
|
||||
#endif /* LIB_ZLIB_CONSTANTS_H */
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* zlib_decompress.c - decompress with a zlib wrapper
|
||||
*
|
||||
* Copyright 2016 Eric Biggers
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following
|
||||
* conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "lib_common.h"
|
||||
#include "zlib_constants.h"
|
||||
|
||||
LIBDEFLATEAPI enum libdeflate_result
|
||||
libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail,
|
||||
size_t *actual_in_nbytes_ret,
|
||||
size_t *actual_out_nbytes_ret)
|
||||
{
|
||||
const u8 *in_next = in;
|
||||
const u8 * const in_end = in_next + in_nbytes;
|
||||
u16 hdr;
|
||||
size_t actual_in_nbytes;
|
||||
size_t actual_out_nbytes;
|
||||
enum libdeflate_result result;
|
||||
|
||||
if (in_nbytes < ZLIB_MIN_OVERHEAD)
|
||||
return LIBDEFLATE_BAD_DATA;
|
||||
|
||||
/* 2 byte header: CMF and FLG */
|
||||
hdr = get_unaligned_be16(in_next);
|
||||
in_next += 2;
|
||||
|
||||
/* FCHECK */
|
||||
if ((hdr % 31) != 0)
|
||||
return LIBDEFLATE_BAD_DATA;
|
||||
|
||||
/* CM */
|
||||
if (((hdr >> 8) & 0xF) != ZLIB_CM_DEFLATE)
|
||||
return LIBDEFLATE_BAD_DATA;
|
||||
|
||||
/* CINFO */
|
||||
if ((hdr >> 12) > ZLIB_CINFO_32K_WINDOW)
|
||||
return LIBDEFLATE_BAD_DATA;
|
||||
|
||||
/* FDICT */
|
||||
if ((hdr >> 5) & 1)
|
||||
return LIBDEFLATE_BAD_DATA;
|
||||
|
||||
/* Compressed data */
|
||||
result = libdeflate_deflate_decompress_ex(d, in_next,
|
||||
in_end - ZLIB_FOOTER_SIZE - in_next,
|
||||
out, out_nbytes_avail,
|
||||
&actual_in_nbytes, actual_out_nbytes_ret);
|
||||
if (result != LIBDEFLATE_SUCCESS)
|
||||
return result;
|
||||
|
||||
if (actual_out_nbytes_ret)
|
||||
actual_out_nbytes = *actual_out_nbytes_ret;
|
||||
else
|
||||
actual_out_nbytes = out_nbytes_avail;
|
||||
|
||||
in_next += actual_in_nbytes;
|
||||
|
||||
/* ADLER32 */
|
||||
if (libdeflate_adler32(1, out, actual_out_nbytes) !=
|
||||
get_unaligned_be32(in_next))
|
||||
return LIBDEFLATE_BAD_DATA;
|
||||
in_next += 4;
|
||||
|
||||
if (actual_in_nbytes_ret)
|
||||
*actual_in_nbytes_ret = in_next - (u8 *)in;
|
||||
|
||||
return LIBDEFLATE_SUCCESS;
|
||||
}
|
||||
|
||||
LIBDEFLATEAPI enum libdeflate_result
|
||||
libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail,
|
||||
size_t *actual_out_nbytes_ret)
|
||||
{
|
||||
return libdeflate_zlib_decompress_ex(d, in, in_nbytes,
|
||||
out, out_nbytes_avail,
|
||||
NULL, actual_out_nbytes_ret);
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
/*
|
||||
* libdeflate.h - public header for libdeflate
|
||||
*/
|
||||
|
||||
#ifndef LIBDEFLATE_H
|
||||
#define LIBDEFLATE_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define LIBDEFLATE_VERSION_MAJOR 1
|
||||
#define LIBDEFLATE_VERSION_MINOR 25
|
||||
#define LIBDEFLATE_VERSION_STRING "1.25"
|
||||
|
||||
/*
|
||||
* Users of libdeflate.dll on Windows can define LIBDEFLATE_DLL to cause
|
||||
* __declspec(dllimport) to be used. This should be done when it's easy to do.
|
||||
* Otherwise it's fine to skip it, since it is a very minor performance
|
||||
* optimization that is irrelevant for most use cases of libdeflate.
|
||||
*/
|
||||
#ifndef LIBDEFLATEAPI
|
||||
# if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
|
||||
# define LIBDEFLATEAPI __declspec(dllimport)
|
||||
# else
|
||||
# define LIBDEFLATEAPI
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Compression */
|
||||
/* ========================================================================== */
|
||||
|
||||
struct libdeflate_compressor;
|
||||
struct libdeflate_options;
|
||||
|
||||
/*
|
||||
* libdeflate_alloc_compressor() allocates a new compressor that supports
|
||||
* DEFLATE, zlib, and gzip compression. 'compression_level' is the compression
|
||||
* level on a zlib-like scale but with a higher maximum value (1 = fastest, 6 =
|
||||
* medium/default, 9 = slow, 12 = slowest). Level 0 is also supported and means
|
||||
* "no compression", specifically "create a valid stream, but only emit
|
||||
* uncompressed blocks" (this will expand the data slightly).
|
||||
*
|
||||
* The return value is a pointer to the new compressor, or NULL if out of memory
|
||||
* or if the compression level is invalid (i.e. outside the range [0, 12]).
|
||||
*
|
||||
* Note: for compression, the sliding window size is defined at compilation time
|
||||
* to 32768, the largest size permissible in the DEFLATE format. It cannot be
|
||||
* changed at runtime.
|
||||
*
|
||||
* A single compressor is not safe to use by multiple threads concurrently.
|
||||
* However, different threads may use different compressors concurrently.
|
||||
*/
|
||||
LIBDEFLATEAPI struct libdeflate_compressor *
|
||||
libdeflate_alloc_compressor(int compression_level);
|
||||
|
||||
/*
|
||||
* Like libdeflate_alloc_compressor(), but adds the 'options' argument.
|
||||
*/
|
||||
LIBDEFLATEAPI struct libdeflate_compressor *
|
||||
libdeflate_alloc_compressor_ex(int compression_level,
|
||||
const struct libdeflate_options *options);
|
||||
|
||||
/*
|
||||
* libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of
|
||||
* data. It attempts to compress 'in_nbytes' bytes of data located at 'in' and
|
||||
* write the result to 'out', which has space for 'out_nbytes_avail' bytes. The
|
||||
* return value is the compressed size in bytes, or 0 if the data could not be
|
||||
* compressed to 'out_nbytes_avail' bytes or fewer.
|
||||
*
|
||||
* If compression is successful, then the output data is guaranteed to be a
|
||||
* valid DEFLATE stream that decompresses to the input data. No other
|
||||
* guarantees are made about the output data. Notably, different versions of
|
||||
* libdeflate can produce different compressed data for the same uncompressed
|
||||
* data, even at the same compression level. Do ***NOT*** do things like
|
||||
* writing tests that compare compressed data to a golden output, as this can
|
||||
* break when libdeflate is updated. (This property isn't specific to
|
||||
* libdeflate; the same is true for zlib and other compression libraries too.)
|
||||
*/
|
||||
LIBDEFLATEAPI size_t
|
||||
libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail);
|
||||
|
||||
/*
|
||||
* libdeflate_deflate_compress_bound() returns a worst-case upper bound on the
|
||||
* number of bytes of compressed data that may be produced by compressing any
|
||||
* buffer of length less than or equal to 'in_nbytes' using
|
||||
* libdeflate_deflate_compress() with the specified compressor. This bound will
|
||||
* necessarily be a number greater than or equal to 'in_nbytes'. It may be an
|
||||
* overestimate of the true upper bound. The return value is guaranteed to be
|
||||
* the same for all invocations with the same compressor and same 'in_nbytes'.
|
||||
*
|
||||
* As a special case, 'compressor' may be NULL. This causes the bound to be
|
||||
* taken across *any* libdeflate_compressor that could ever be allocated with
|
||||
* this build of the library, with any options.
|
||||
*
|
||||
* Note that this function is not necessary in many applications. With
|
||||
* block-based compression, it is usually preferable to separately store the
|
||||
* uncompressed size of each block and to store any blocks that did not compress
|
||||
* to less than their original size uncompressed. In that scenario, there is no
|
||||
* need to know the worst-case compressed size, since the maximum number of
|
||||
* bytes of compressed data that may be used would always be one less than the
|
||||
* input length. You can just pass a buffer of that size to
|
||||
* libdeflate_deflate_compress() and store the data uncompressed if
|
||||
* libdeflate_deflate_compress() returns 0, indicating that the compressed data
|
||||
* did not fit into the provided output buffer.
|
||||
*/
|
||||
LIBDEFLATEAPI size_t
|
||||
libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
|
||||
size_t in_nbytes);
|
||||
|
||||
/*
|
||||
* Like libdeflate_deflate_compress(), but uses the zlib wrapper format instead
|
||||
* of raw DEFLATE.
|
||||
*/
|
||||
LIBDEFLATEAPI size_t
|
||||
libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail);
|
||||
|
||||
/*
|
||||
* Like libdeflate_deflate_compress_bound(), but assumes the data will be
|
||||
* compressed with libdeflate_zlib_compress() rather than with
|
||||
* libdeflate_deflate_compress().
|
||||
*/
|
||||
LIBDEFLATEAPI size_t
|
||||
libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
|
||||
size_t in_nbytes);
|
||||
|
||||
/*
|
||||
* Like libdeflate_deflate_compress(), but uses the gzip wrapper format instead
|
||||
* of raw DEFLATE.
|
||||
*/
|
||||
LIBDEFLATEAPI size_t
|
||||
libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail);
|
||||
|
||||
/*
|
||||
* Like libdeflate_deflate_compress_bound(), but assumes the data will be
|
||||
* compressed with libdeflate_gzip_compress() rather than with
|
||||
* libdeflate_deflate_compress().
|
||||
*/
|
||||
LIBDEFLATEAPI size_t
|
||||
libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
|
||||
size_t in_nbytes);
|
||||
|
||||
/*
|
||||
* libdeflate_free_compressor() frees a compressor that was allocated with
|
||||
* libdeflate_alloc_compressor(). If a NULL pointer is passed in, no action is
|
||||
* taken.
|
||||
*/
|
||||
LIBDEFLATEAPI void
|
||||
libdeflate_free_compressor(struct libdeflate_compressor *compressor);
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Decompression */
|
||||
/* ========================================================================== */
|
||||
|
||||
struct libdeflate_decompressor;
|
||||
struct libdeflate_options;
|
||||
|
||||
/*
|
||||
* libdeflate_alloc_decompressor() allocates a new decompressor that can be used
|
||||
* for DEFLATE, zlib, and gzip decompression. The return value is a pointer to
|
||||
* the new decompressor, or NULL if out of memory.
|
||||
*
|
||||
* This function takes no parameters, and the returned decompressor is valid for
|
||||
* decompressing data that was compressed at any compression level and with any
|
||||
* sliding window size.
|
||||
*
|
||||
* A single decompressor is not safe to use by multiple threads concurrently.
|
||||
* However, different threads may use different decompressors concurrently.
|
||||
*/
|
||||
LIBDEFLATEAPI struct libdeflate_decompressor *
|
||||
libdeflate_alloc_decompressor(void);
|
||||
|
||||
/*
|
||||
* Like libdeflate_alloc_decompressor(), but adds the 'options' argument.
|
||||
*/
|
||||
LIBDEFLATEAPI struct libdeflate_decompressor *
|
||||
libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
|
||||
|
||||
/*
|
||||
* Result of a call to libdeflate_deflate_decompress(),
|
||||
* libdeflate_zlib_decompress(), or libdeflate_gzip_decompress().
|
||||
*/
|
||||
enum libdeflate_result {
|
||||
/* Decompression was successful. */
|
||||
LIBDEFLATE_SUCCESS = 0,
|
||||
|
||||
/* Decompression failed because the compressed data was invalid,
|
||||
* corrupt, or otherwise unsupported. */
|
||||
LIBDEFLATE_BAD_DATA = 1,
|
||||
|
||||
/* A NULL 'actual_out_nbytes_ret' was provided, but the data would have
|
||||
* decompressed to fewer than 'out_nbytes_avail' bytes. */
|
||||
LIBDEFLATE_SHORT_OUTPUT = 2,
|
||||
|
||||
/* The data would have decompressed to more than 'out_nbytes_avail'
|
||||
* bytes. */
|
||||
LIBDEFLATE_INSUFFICIENT_SPACE = 3,
|
||||
};
|
||||
|
||||
/*
|
||||
* libdeflate_deflate_decompress() decompresses a DEFLATE stream from the buffer
|
||||
* 'in' with compressed size up to 'in_nbytes' bytes. The uncompressed data is
|
||||
* written to 'out', a buffer with size 'out_nbytes_avail' bytes. If
|
||||
* decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned. Otherwise,
|
||||
* a nonzero result code such as LIBDEFLATE_BAD_DATA is returned, and the
|
||||
* contents of the output buffer are undefined.
|
||||
*
|
||||
* Decompression stops at the end of the DEFLATE stream (as indicated by the
|
||||
* BFINAL flag), even if it is actually shorter than 'in_nbytes' bytes.
|
||||
*
|
||||
* libdeflate_deflate_decompress() can be used in cases where the actual
|
||||
* uncompressed size is known (recommended) or unknown (not recommended):
|
||||
*
|
||||
* - If the actual uncompressed size is known, then pass the actual
|
||||
* uncompressed size as 'out_nbytes_avail' and pass NULL for
|
||||
* 'actual_out_nbytes_ret'. This makes libdeflate_deflate_decompress() fail
|
||||
* with LIBDEFLATE_SHORT_OUTPUT if the data decompressed to fewer than the
|
||||
* specified number of bytes.
|
||||
*
|
||||
* - If the actual uncompressed size is unknown, then provide a non-NULL
|
||||
* 'actual_out_nbytes_ret' and provide a buffer with some size
|
||||
* 'out_nbytes_avail' that you think is large enough to hold all the
|
||||
* uncompressed data. In this case, if the data decompresses to less than
|
||||
* or equal to 'out_nbytes_avail' bytes, then
|
||||
* libdeflate_deflate_decompress() will write the actual uncompressed size
|
||||
* to *actual_out_nbytes_ret and return 0 (LIBDEFLATE_SUCCESS). Otherwise,
|
||||
* it will return LIBDEFLATE_INSUFFICIENT_SPACE if the provided buffer was
|
||||
* not large enough but no other problems were encountered, or another
|
||||
* nonzero result code if decompression failed for another reason.
|
||||
*/
|
||||
LIBDEFLATEAPI enum libdeflate_result
|
||||
libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail,
|
||||
size_t *actual_out_nbytes_ret);
|
||||
|
||||
/*
|
||||
* Like libdeflate_deflate_decompress(), but adds the 'actual_in_nbytes_ret'
|
||||
* argument. If decompression succeeds and 'actual_in_nbytes_ret' is not NULL,
|
||||
* then the actual compressed size of the DEFLATE stream (aligned to the next
|
||||
* byte boundary) is written to *actual_in_nbytes_ret.
|
||||
*/
|
||||
LIBDEFLATEAPI enum libdeflate_result
|
||||
libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail,
|
||||
size_t *actual_in_nbytes_ret,
|
||||
size_t *actual_out_nbytes_ret);
|
||||
|
||||
/*
|
||||
* Like libdeflate_deflate_decompress(), but assumes the zlib wrapper format
|
||||
* instead of raw DEFLATE.
|
||||
*
|
||||
* Decompression will stop at the end of the zlib stream, even if it is shorter
|
||||
* than 'in_nbytes'. If you need to know exactly where the zlib stream ended,
|
||||
* use libdeflate_zlib_decompress_ex().
|
||||
*/
|
||||
LIBDEFLATEAPI enum libdeflate_result
|
||||
libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail,
|
||||
size_t *actual_out_nbytes_ret);
|
||||
|
||||
/*
|
||||
* Like libdeflate_zlib_decompress(), but adds the 'actual_in_nbytes_ret'
|
||||
* argument. If 'actual_in_nbytes_ret' is not NULL and the decompression
|
||||
* succeeds (indicating that the first zlib-compressed stream in the input
|
||||
* buffer was decompressed), then the actual number of input bytes consumed is
|
||||
* written to *actual_in_nbytes_ret.
|
||||
*/
|
||||
LIBDEFLATEAPI enum libdeflate_result
|
||||
libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail,
|
||||
size_t *actual_in_nbytes_ret,
|
||||
size_t *actual_out_nbytes_ret);
|
||||
|
||||
/*
|
||||
* Like libdeflate_deflate_decompress(), but assumes the gzip wrapper format
|
||||
* instead of raw DEFLATE.
|
||||
*
|
||||
* If multiple gzip-compressed members are concatenated, then only the first
|
||||
* will be decompressed. Use libdeflate_gzip_decompress_ex() if you need
|
||||
* multi-member support.
|
||||
*/
|
||||
LIBDEFLATEAPI enum libdeflate_result
|
||||
libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail,
|
||||
size_t *actual_out_nbytes_ret);
|
||||
|
||||
/*
|
||||
* Like libdeflate_gzip_decompress(), but adds the 'actual_in_nbytes_ret'
|
||||
* argument. If 'actual_in_nbytes_ret' is not NULL and the decompression
|
||||
* succeeds (indicating that the first gzip-compressed member in the input
|
||||
* buffer was decompressed), then the actual number of input bytes consumed is
|
||||
* written to *actual_in_nbytes_ret.
|
||||
*/
|
||||
LIBDEFLATEAPI enum libdeflate_result
|
||||
libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail,
|
||||
size_t *actual_in_nbytes_ret,
|
||||
size_t *actual_out_nbytes_ret);
|
||||
|
||||
/*
|
||||
* libdeflate_free_decompressor() frees a decompressor that was allocated with
|
||||
* libdeflate_alloc_decompressor(). If a NULL pointer is passed in, no action
|
||||
* is taken.
|
||||
*/
|
||||
LIBDEFLATEAPI void
|
||||
libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Checksums */
|
||||
/* ========================================================================== */
|
||||
|
||||
/*
|
||||
* libdeflate_adler32() updates a running Adler-32 checksum with 'len' bytes of
|
||||
* data and returns the updated checksum. When starting a new checksum, the
|
||||
* required initial value for 'adler' is 1. This value is also returned when
|
||||
* 'buffer' is specified as NULL.
|
||||
*/
|
||||
LIBDEFLATEAPI uint32_t
|
||||
libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
|
||||
|
||||
|
||||
/*
|
||||
* libdeflate_crc32() updates a running CRC-32 checksum with 'len' bytes of data
|
||||
* and returns the updated checksum. When starting a new checksum, the required
|
||||
* initial value for 'crc' is 0. This value is also returned when 'buffer' is
|
||||
* specified as NULL.
|
||||
*/
|
||||
LIBDEFLATEAPI uint32_t
|
||||
libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
|
||||
|
||||
/* ========================================================================== */
|
||||
/* Custom memory allocator */
|
||||
/* ========================================================================== */
|
||||
|
||||
/*
|
||||
* Install a custom memory allocator which libdeflate will use for all memory
|
||||
* allocations by default. 'malloc_func' is a function that must behave like
|
||||
* malloc(), and 'free_func' is a function that must behave like free().
|
||||
*
|
||||
* The per-(de)compressor custom memory allocator that can be specified in
|
||||
* 'struct libdeflate_options' takes priority over this.
|
||||
*
|
||||
* This doesn't affect the free() function that will be used to free
|
||||
* (de)compressors that were already in existence when this is called.
|
||||
*/
|
||||
LIBDEFLATEAPI void
|
||||
libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
|
||||
void (*free_func)(void *));
|
||||
|
||||
/*
|
||||
* Advanced options. This is the options structure that
|
||||
* libdeflate_alloc_compressor_ex() and libdeflate_alloc_decompressor_ex()
|
||||
* require. Most users won't need this and should just use the non-"_ex"
|
||||
* functions instead. If you do need this, it should be initialized like this:
|
||||
*
|
||||
* struct libdeflate_options options;
|
||||
*
|
||||
* memset(&options, 0, sizeof(options));
|
||||
* options.sizeof_options = sizeof(options);
|
||||
* // Then set the fields that you need to override the defaults for.
|
||||
*/
|
||||
struct libdeflate_options {
|
||||
|
||||
/*
|
||||
* This field must be set to the struct size. This field exists for
|
||||
* extensibility, so that fields can be appended to this struct in
|
||||
* future versions of libdeflate while still supporting old binaries.
|
||||
*/
|
||||
size_t sizeof_options;
|
||||
|
||||
/*
|
||||
* An optional custom memory allocator to use for this (de)compressor.
|
||||
* 'malloc_func' must be a function that behaves like malloc(), and
|
||||
* 'free_func' must be a function that behaves like free().
|
||||
*
|
||||
* This is useful in cases where a process might have multiple users of
|
||||
* libdeflate who want to use different memory allocators. For example,
|
||||
* a library might want to use libdeflate with a custom memory allocator
|
||||
* without interfering with user code that might use libdeflate too.
|
||||
*
|
||||
* This takes priority over the "global" memory allocator (which by
|
||||
* default is malloc() and free(), but can be changed by
|
||||
* libdeflate_set_memory_allocator()). Moreover, libdeflate will never
|
||||
* call the "global" memory allocator if a per-(de)compressor custom
|
||||
* allocator is always given.
|
||||
*/
|
||||
void *(*malloc_func)(size_t);
|
||||
void (*free_func)(void *);
|
||||
};
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* LIBDEFLATE_H */
|
||||
@@ -0,0 +1,25 @@
|
||||
/* Wraps libdeflate_deflate_decompress with SEH to catch access violations.
|
||||
* libdeflate's fast path on 32-bit can crash on malformed input despite
|
||||
* SAFETY_CHECKs. This wrapper catches the crash and returns an error code. */
|
||||
|
||||
#include "libdeflate.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
|
||||
__declspec(dllexport) int safe_deflate_decompress(
|
||||
struct libdeflate_decompressor *d,
|
||||
const void *in, size_t in_nbytes,
|
||||
void *out, size_t out_nbytes_avail,
|
||||
size_t *actual_out_nbytes_ret)
|
||||
{
|
||||
int result;
|
||||
__try {
|
||||
result = libdeflate_deflate_decompress(d, in, in_nbytes,
|
||||
out, out_nbytes_avail, actual_out_nbytes_ret);
|
||||
} __except(EXCEPTION_EXECUTE_HANDLER) {
|
||||
result = 1; /* LIBDEFLATE_BAD_DATA */
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,15 @@
|
||||
/* malloc/free stubs for libdeflate using the Windows process heap. */
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
__declspec(dllimport) void *__stdcall GetProcessHeap(void);
|
||||
__declspec(dllimport) void *__stdcall HeapAlloc(void *hHeap, unsigned long dwFlags, size_t dwBytes);
|
||||
__declspec(dllimport) int __stdcall HeapFree(void *hHeap, unsigned long dwFlags, void *lpMem);
|
||||
|
||||
void *malloc(size_t size) {
|
||||
return HeapAlloc(GetProcessHeap(), 0, size);
|
||||
}
|
||||
|
||||
void free(void *ptr) {
|
||||
if (ptr) HeapFree(GetProcessHeap(), 0, ptr);
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
#pragma once
|
||||
#include <stddef.h>
|
||||
extern void *malloc(size_t size);
|
||||
extern void free(void *ptr);
|
||||
@@ -0,0 +1,3 @@
|
||||
#pragma once
|
||||
/* Stub: libdeflate doesn't use setjmp, but MSVC intrin.h includes it */
|
||||
typedef int jmp_buf[16];
|
||||
@@ -0,0 +1,2 @@
|
||||
#pragma once
|
||||
/* libdeflate only uses stdio.h in debug paths — stub it out */
|
||||
@@ -0,0 +1,4 @@
|
||||
#pragma once
|
||||
#include <stddef.h>
|
||||
extern void *malloc(size_t size);
|
||||
extern void free(void *ptr);
|
||||
@@ -0,0 +1,6 @@
|
||||
#pragma once
|
||||
#include <stddef.h>
|
||||
void *memcpy(void *dest, const void *src, size_t n);
|
||||
void *memmove(void *dest, const void *src, size_t n);
|
||||
void *memset(void *s, int c, size_t n);
|
||||
int memcmp(const void *s1, const void *s2, size_t n);
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,709 @@
|
||||
//! SSE math implementations for silicon module.
|
||||
//!
|
||||
//! Pure math functions extracted from silicon.zig for standalone compilation.
|
||||
//! Used by both the DLL (via silicon.zig wrappers) and the bench harness.
|
||||
//! All functions use C ABI (export fn) — callers handle CC translation.
|
||||
//!
|
||||
//! Compiled with SSE4.1+FMA+AVX. Uses @Vector(4, f32) and @mulAdd throughout.
|
||||
|
||||
const std = @import("std");
|
||||
const V4 = @Vector(4, f32);
|
||||
const CC = std.builtin.CallingConvention;
|
||||
const TC: CC = .{ .x86_thiscall = .{} };
|
||||
const FC: CC = .{ .x86_fastcall = .{} };
|
||||
const SC: CC = .{ .x86_stdcall = .{} };
|
||||
|
||||
inline fn loadV4(ptr: u32) V4 {
|
||||
return @as(*align(1) const V4, @ptrFromInt(ptr)).*;
|
||||
}
|
||||
|
||||
inline fn loadV3_1(ptr: u32) V4 {
|
||||
const p: [*]const f32 = @ptrFromInt(ptr);
|
||||
return V4{ p[0], p[1], p[2], 1.0 };
|
||||
}
|
||||
|
||||
inline fn dot3v(a: V4, b: V4) f32 {
|
||||
return @mulAdd(f32, a[2], b[2], @mulAdd(f32, a[1], b[1], a[0] * b[0]));
|
||||
}
|
||||
|
||||
inline fn dot4v(a: V4, b: V4) f32 {
|
||||
return @mulAdd(f32, a[3], b[3], @mulAdd(f32, a[2], b[2], @mulAdd(f32, a[1], b[1], a[0] * b[0])));
|
||||
}
|
||||
|
||||
/// Fast reciprocal via vrcpss + Newton-Raphson. ~5 cycles, ~23-bit accuracy (vs vdivss ~14 cycles).
|
||||
inline fn fastRecip(x: f32) f32 {
|
||||
var rcp: f32 = undefined;
|
||||
// vrcpss: 12-bit approximation of 1/x
|
||||
rcp = asm ("vrcpss %[x], %[x], %[rcp]"
|
||||
: [rcp] "=x" (-> f32),
|
||||
: [x] "x" (x),
|
||||
);
|
||||
// Newton-Raphson: rcp = rcp * (2 - x * rcp)
|
||||
return rcp * @mulAdd(f32, -x, rcp, 2.0);
|
||||
}
|
||||
|
||||
/// Round float to nearest integer via cvtss2si (uses MXCSR rounding mode, default = round-to-nearest).
|
||||
/// Single instruction, replaces @round + @intFromFloat (~6 instructions).
|
||||
inline fn cvtss2si(x: f32) i32 {
|
||||
return asm ("vcvtss2si %[x], %[out]"
|
||||
: [out] "=r" (-> i32),
|
||||
: [x] "x" (x),
|
||||
);
|
||||
}
|
||||
|
||||
// --- 0x4549C0: normalizeVec3 (137K/7.5s) ---
|
||||
// Naked thiscall: ECX=vec, [ESP+4]=length_bits. RET 4. Original: 38 bytes.
|
||||
// rcpss + NR for fast reciprocal, then 3 multiplies.
|
||||
pub fn si_normalizeVec3() callconv(.naked) void {
|
||||
asm volatile (
|
||||
// xmm0 = 1.0 / length (via rcpss + Newton-Raphson)
|
||||
\\vmovss 4(%%esp), %%xmm0
|
||||
\\vrcpss %%xmm0, %%xmm0, %%xmm1
|
||||
\\vmulss %%xmm1, %%xmm0, %%xmm2
|
||||
\\vaddss %%xmm1, %%xmm1, %%xmm0
|
||||
\\vfnmadd231ss %%xmm1, %%xmm2, %%xmm0
|
||||
// xmm0 = refined 1/length. Multiply 3 components.
|
||||
\\vmulss (%%ecx), %%xmm0, %%xmm1
|
||||
\\vmovss %%xmm1, (%%ecx)
|
||||
\\vmulss 4(%%ecx), %%xmm0, %%xmm1
|
||||
\\vmovss %%xmm1, 4(%%ecx)
|
||||
\\vmulss 8(%%ecx), %%xmm0, %%xmm1
|
||||
\\vmovss %%xmm1, 8(%%ecx)
|
||||
\\ret $4
|
||||
);
|
||||
}
|
||||
|
||||
// --- 0x7BAE60: mulMat3x4 ---
|
||||
// out = A * B (3x4 layout: 3x3 rotation + 3 translation)
|
||||
// Layout: [r0c0 r0c1 r0c2 | r1c0 r1c1 r1c2 | r2c0 r2c1 r2c2 | tx ty tz]
|
||||
// V4 per row: broadcast b[row*3+k], multiply with a's columns, accumulate.
|
||||
pub fn si_mulMat3x4(out: u32, a_ptr: u32, b_ptr: u32) callconv(FC) u32 {
|
||||
const dst: [*]f32 = @ptrFromInt(out);
|
||||
const aa: [*]const f32 = @ptrFromInt(a_ptr);
|
||||
const b: [*]const f32 = @ptrFromInt(b_ptr);
|
||||
|
||||
// a is row-major 3x3. a[0..2] = row 0, a[3..5] = row 1, a[6..8] = row 2.
|
||||
// The formula: dst[row*3+col] = a[col]*b[row*3] + a[col+3]*b[row*3+1] + a[col+6]*b[row*3+2]
|
||||
// This means: for each output row, we broadcast b's row elements and dot with a's "columns"
|
||||
// a's "column k" is {a[k], a[k+3], a[k+6]} — that's picking one element from each of a's rows.
|
||||
|
||||
// Load a's columns (stride 3)
|
||||
const ac0 = V4{ aa[0], aa[3], aa[6], aa[9] }; // col 0 of a (+ translation x)
|
||||
const ac1 = V4{ aa[1], aa[4], aa[7], aa[10] }; // col 1 of a (+ translation y)
|
||||
const ac2 = V4{ aa[2], aa[5], aa[8], aa[11] }; // col 2 of a (+ translation z)
|
||||
|
||||
// Rotation: 3 output rows
|
||||
inline for (0..3) |row| {
|
||||
const br0: V4 = @splat(b[row * 3]);
|
||||
const br1: V4 = @splat(b[row * 3 + 1]);
|
||||
const br2: V4 = @splat(b[row * 3 + 2]);
|
||||
const result = @mulAdd(V4, ac2, br2, @mulAdd(V4, ac1, br1, ac0 * br0));
|
||||
dst[row * 3] = result[0];
|
||||
dst[row * 3 + 1] = result[1];
|
||||
dst[row * 3 + 2] = result[2];
|
||||
}
|
||||
|
||||
// Translation: dst[9+i] = a_trans dot b_col_i + b_trans[i]
|
||||
// = ac0[3]*b[i] + ac1[3]*b[i+3] + ac2[3]*b[i+6] + b[9+i]
|
||||
// Using the V4 approach: broadcast b elements, same ac columns, take lane 3 + add b_trans
|
||||
// Translation: scalar @mulAdd (b's columns don't align for V4)
|
||||
inline for (0..3) |col| {
|
||||
dst[9 + col] = @mulAdd(f32, aa[11], b[col + 6], @mulAdd(f32, aa[10], b[col + 3], @mulAdd(f32, aa[9], b[col], b[9 + col])));
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- 0x7BDDB0: rotateMatByQuat ---
|
||||
// builds rotation matrix from quaternion, multiplies with existing 4x4 matrix
|
||||
// Uses V4 for the matrix multiply (same pattern as bone_sse)
|
||||
pub fn si_rotateMatByQuat(mat: u32, quat: u32) callconv(TC) u32 {
|
||||
const q: [*]const f32 = @ptrFromInt(quat);
|
||||
const x = q[0]; const y = q[1]; const z = q[2]; const w = q[3];
|
||||
const x2 = x + x; const y2 = y + y; const z2 = z + z;
|
||||
const xx = x * x2; const xy = x * y2; const xz = x * z2;
|
||||
const yy = y * y2; const yz = y * z2; const zz = z * z2;
|
||||
const wx = w * x2; const wy = w * y2; const wz = w * z2;
|
||||
|
||||
const q0 = V4{ 1.0 - (yy + zz), xy + wz, xz - wy, 0 };
|
||||
const q1 = V4{ xy - wz, 1.0 - (xx + zz), yz + wx, 0 };
|
||||
const q2 = V4{ xz + wy, yz - wx, 1.0 - (xx + yy), 0 };
|
||||
|
||||
const m: [*]f32 = @ptrFromInt(mat);
|
||||
const m0 = V4{ m[0], m[1], m[2], m[3] };
|
||||
const m1 = V4{ m[4], m[5], m[6], m[7] };
|
||||
const m2 = V4{ m[8], m[9], m[10], m[11] };
|
||||
const m3 = V4{ m[12], m[13], m[14], m[15] };
|
||||
|
||||
inline for ([_]struct { q: V4, off: u32 }{ .{ .q = q0, .off = 0 }, .{ .q = q1, .off = 4 }, .{ .q = q2, .off = 8 } }) |r| {
|
||||
const row = @mulAdd(V4, @as(V4, @splat(r.q[2])), m2, @mulAdd(V4, @as(V4, @splat(r.q[1])), m1, @as(V4, @splat(r.q[0])) * m0));
|
||||
m[r.off] = row[0]; m[r.off + 1] = row[1]; m[r.off + 2] = row[2]; m[r.off + 3] = row[3];
|
||||
}
|
||||
// Row 3 unchanged (identity row)
|
||||
m[12] = m3[0]; m[13] = m3[1]; m[14] = m3[2]; m[15] = m3[3];
|
||||
return mat;
|
||||
}
|
||||
|
||||
// --- 0x7BB860: createRotMat3x4 ---
|
||||
// Rodrigues rotation matrix, 3x4 layout. Uses @mulAdd for all 9 entries.
|
||||
pub fn si_createRotMat3x4(out: u32, axis_ptr: u32, angle_bits: u32, is_normalized: u32) callconv(FC) u32 {
|
||||
const m: [*]f32 = @ptrFromInt(out);
|
||||
const ax: [*]const f32 = @ptrFromInt(axis_ptr);
|
||||
var x = ax[0]; var y = ax[1]; var z = ax[2];
|
||||
if (is_normalized == 0) {
|
||||
const len = @sqrt(@mulAdd(f32, z, z, @mulAdd(f32, y, y, x * x)));
|
||||
if (len > 1.0e-20) { const inv = 1.0 / len; x *= inv; y *= inv; z *= inv; }
|
||||
}
|
||||
const angle: f32 = @bitCast(angle_bits);
|
||||
// x87 FSINCOS: single instruction computes both sin and cos
|
||||
// FSINCOS tested at 148cy — x87 microcode is slow on modern CPUs.
|
||||
// Library sinf+cosf (~35cy each = 70cy total) is 2x faster.
|
||||
const c = @cos(angle); const s = @sin(angle); const t = 1.0 - c;
|
||||
m[0] = @mulAdd(f32, t * x, x, c); m[1] = @mulAdd(f32, s, z, t * x * y); m[2] = @mulAdd(f32, -s, y, t * x * z);
|
||||
m[3] = @mulAdd(f32, -s, z, t * x * y); m[4] = @mulAdd(f32, t * y, y, c); m[5] = @mulAdd(f32, s, x, t * y * z);
|
||||
m[6] = @mulAdd(f32, s, y, t * x * z); m[7] = @mulAdd(f32, -s, x, t * y * z); m[8] = @mulAdd(f32, t * z, z, c);
|
||||
m[9] = 0; m[10] = 0; m[11] = 0;
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- 0x6329E0: distanceToPlane (525K/7.5s) ---
|
||||
// __fastcall(ECX=point, EDX=plane, stack=direction), returns f64 via ST(0), RET 0x4.
|
||||
pub fn si_distanceToPlane(point: u32, plane: u32, direction: u32) callconv(FC) f64 {
|
||||
const p: [*]const f32 = @ptrFromInt(point);
|
||||
const pl: [*]const f32 = @ptrFromInt(plane);
|
||||
const dir: [*]const f32 = @ptrFromInt(direction);
|
||||
const dot1 = @mulAdd(f32, p[2], pl[2], @mulAdd(f32, p[1], pl[1], @mulAdd(f32, p[0], pl[0], pl[3])));
|
||||
const dot2 = @mulAdd(f32, dir[2], pl[2], @mulAdd(f32, dir[1], pl[1], dir[0] * pl[0]));
|
||||
if (@abs(dot2) < 1.0e-20) return 0.0;
|
||||
return @as(f64, dot1) / @as(f64, dot2);
|
||||
}
|
||||
|
||||
|
||||
// --- 0x686C20: classifyPointFrustum (3.2M/7.5s) ---
|
||||
// Tests point against 6 frustum planes, produces 6-bit bitmask.
|
||||
// Scalar @mulAdd dot4 per plane — the FMA chain has best throughput for this pattern.
|
||||
// Tried: V4 batch 4 planes (gather kills it), V4 hsum (shuffle overhead kills it).
|
||||
pub fn si_classifyPointFrustum(planes_ptr: u32, point: u32, out_mask: u32) callconv(TC) u32 {
|
||||
const mask: *u32 = @ptrFromInt(out_mask);
|
||||
const pt = loadV3_1(point);
|
||||
var bits: u32 = 0;
|
||||
inline for (0..6) |i| {
|
||||
const pl = loadV4(planes_ptr + i * 16);
|
||||
const dist = dot4v(pt, pl);
|
||||
// Branchless: extract sign bit via bit cast
|
||||
bits |= (@as(u32, @bitCast(dist)) >> 31) << i;
|
||||
}
|
||||
mask.* = bits;
|
||||
return planes_ptr;
|
||||
}
|
||||
|
||||
// --- 0x6DC5A0: checkBoxLineIntersect (2.7M/7.5s) ---
|
||||
// Slab AABB test. Branchless min/max for t0/t1 swap and tmin/tmax accumulation.
|
||||
pub fn si_checkBoxLineIntersect(box_ptr: u32, line_start: u32, line_end: u32) callconv(FC) u32 {
|
||||
const bmin: [*]const f32 = @ptrFromInt(box_ptr);
|
||||
const bmax: [*]const f32 = @ptrFromInt(box_ptr + 0xC);
|
||||
const start: [*]const f32 = @ptrFromInt(line_start);
|
||||
const end_pt: [*]const f32 = @ptrFromInt(line_end);
|
||||
var tmin: f32 = 0.0;
|
||||
var tmax: f32 = 1.0;
|
||||
inline for (0..3) |i| {
|
||||
const dir = end_pt[i] - start[i];
|
||||
if (@abs(dir) < 1.0e-20) {
|
||||
if (start[i] < bmin[i] or start[i] > bmax[i]) return 0;
|
||||
} else {
|
||||
const inv_dir = 1.0 / dir;
|
||||
const ta = (bmin[i] - start[i]) * inv_dir;
|
||||
const tb = (bmax[i] - start[i]) * inv_dir;
|
||||
// Branchless swap: vminss/vmaxss instead of compare+branch
|
||||
tmin = @max(tmin, @min(ta, tb));
|
||||
tmax = @min(tmax, @max(ta, tb));
|
||||
if (tmin > tmax) return 0;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
// --- 0x6869C0: testOBBFrustum ---
|
||||
// Tests OBB against 6 frustum planes. Uses V4 for corner transform and plane test.
|
||||
pub fn si_testOBBFrustum(planes_ptr: u32, aabb_ptr: u32, rot_ptr: u32, trans_ptr: u32) callconv(TC) u32 {
|
||||
const aabb: [*]const f32 = @ptrFromInt(aabb_ptr);
|
||||
const rot: [*]const f32 = @ptrFromInt(rot_ptr);
|
||||
const t: [*]const f32 = @ptrFromInt(trans_ptr);
|
||||
|
||||
// Rotation columns as V4 (xyz + 0 for w)
|
||||
const rc0 = V4{ rot[0], rot[1], rot[2], 0 };
|
||||
const rc1 = V4{ rot[3], rot[4], rot[5], 0 };
|
||||
const rc2 = V4{ rot[6], rot[7], rot[8], 0 };
|
||||
const tv = V4{ t[0], t[1], t[2], 1.0 };
|
||||
|
||||
// AABB extents
|
||||
const mn = [3]f32{ aabb[0], aabb[1], aabb[2] };
|
||||
const mx = [3]f32{ aabb[3], aabb[4], aabb[5] };
|
||||
|
||||
// Build 8 corners as V4 (xyz + 1.0 for plane dot w/ d term)
|
||||
var corners: [8]V4 = undefined;
|
||||
inline for (0..8) |i| {
|
||||
const lx: f32 = if (i & 1 != 0) mx[0] else mn[0];
|
||||
const ly: f32 = if (i & 2 != 0) mx[1] else mn[1];
|
||||
const lz: f32 = if (i & 4 != 0) mx[2] else mn[2];
|
||||
corners[i] = @mulAdd(V4, @as(V4, @splat(lz)), rc2, @mulAdd(V4, @as(V4, @splat(ly)), rc1, @mulAdd(V4, @as(V4, @splat(lx)), rc0, tv)));
|
||||
}
|
||||
|
||||
// Extract x/y/z/w components across corners for batched plane tests
|
||||
// Group A: corners 0-3, Group B: corners 4-7
|
||||
const cx_a = V4{ corners[0][0], corners[1][0], corners[2][0], corners[3][0] };
|
||||
const cy_a = V4{ corners[0][1], corners[1][1], corners[2][1], corners[3][1] };
|
||||
const cz_a = V4{ corners[0][2], corners[1][2], corners[2][2], corners[3][2] };
|
||||
|
||||
const cx_b = V4{ corners[4][0], corners[5][0], corners[6][0], corners[7][0] };
|
||||
const cy_b = V4{ corners[4][1], corners[5][1], corners[6][1], corners[7][1] };
|
||||
const cz_b = V4{ corners[4][2], corners[5][2], corners[6][2], corners[7][2] };
|
||||
|
||||
// Test each plane: compute 4 dots at a time, branchless sign check
|
||||
inline for (0..6) |p| {
|
||||
const pl = loadV4(planes_ptr + p * 16);
|
||||
const pnx: V4 = @splat(pl[0]);
|
||||
const pny: V4 = @splat(pl[1]);
|
||||
const pnz: V4 = @splat(pl[2]);
|
||||
const pd: V4 = @splat(pl[3]);
|
||||
|
||||
// 4 dots for corners 0-3
|
||||
const da = @mulAdd(V4, cz_a, pnz, @mulAdd(V4, cy_a, pny, @mulAdd(V4, cx_a, pnx, pd)));
|
||||
// 4 dots for corners 4-7
|
||||
const db = @mulAdd(V4, cz_b, pnz, @mulAdd(V4, cy_b, pny, @mulAdd(V4, cx_b, pnx, pd)));
|
||||
|
||||
// All outside if all 8 distances are negative
|
||||
// Branchless: extract sign bits via comparison
|
||||
const neg_a = da < @as(V4, @splat(@as(f32, 0)));
|
||||
const neg_b = db < @as(V4, @splat(@as(f32, 0)));
|
||||
const mask_a: u4 = @bitCast(neg_a);
|
||||
const mask_b: u4 = @bitCast(neg_b);
|
||||
if (mask_a == 0xF and mask_b == 0xF) return 0;
|
||||
}
|
||||
return 3;
|
||||
}
|
||||
|
||||
// --- 0x686B80: testSphereFrustum (375K/7.5s) ---
|
||||
// Zig thiscall: naked asm tested at 10cy (vhaddps slow), Zig dot4v at 8cy.
|
||||
pub fn si_testSphereFrustum(planes_ptr: u32, sphere: u32) callconv(TC) u32 {
|
||||
const s: [*]const f32 = @ptrFromInt(sphere);
|
||||
const center = V4{ s[0], s[1], s[2], 1.0 };
|
||||
const r = s[3];
|
||||
inline for (0..6) |i| {
|
||||
const pl = loadV4(planes_ptr + i * 16);
|
||||
if (dot4v(center, pl) < -r) return 0;
|
||||
}
|
||||
return 3;
|
||||
}
|
||||
|
||||
|
||||
// --- 0x7C0570: quatSlerp ---
|
||||
// V4 for final blend, @mulAdd for dot product
|
||||
pub fn si_quatSlerp(out: u32, a_ptr: u32, t_bits: u32, b_ptr: u32) callconv(FC) u32 {
|
||||
const dst: [*]f32 = @ptrFromInt(out);
|
||||
const av = loadV4(a_ptr);
|
||||
const bv = loadV4(b_ptr);
|
||||
const tt: f32 = @bitCast(t_bits);
|
||||
var dot_val = dot4v(av, bv);
|
||||
var sign: f32 = 1.0;
|
||||
if (dot_val < 0) { dot_val = -dot_val; sign = -1.0; }
|
||||
var s0: f32 = undefined;
|
||||
var s1: f32 = undefined;
|
||||
if (dot_val > 0.9995) {
|
||||
s0 = 1.0 - tt;
|
||||
s1 = tt * sign;
|
||||
} else {
|
||||
const theta = std.math.acos(dot_val);
|
||||
const sin_theta = @sin(theta);
|
||||
const inv_sin = 1.0 / sin_theta;
|
||||
s0 = @sin((1.0 - tt) * theta) * inv_sin;
|
||||
s1 = @sin(tt * theta) * inv_sin * sign;
|
||||
}
|
||||
const result = @mulAdd(V4, @as(V4, @splat(s1)), bv, @as(V4, @splat(s0)) * av);
|
||||
dst[0] = result[0]; dst[1] = result[1]; dst[2] = result[2]; dst[3] = result[3];
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- 0x699330: isPointInsideBounds (1.7M/7.5s) ---
|
||||
// __fastcall(ECX=a, EDX=b), returns u32.
|
||||
pub fn si_isPointInsideBounds(a: u32, b: u32) callconv(FC) u32 {
|
||||
const va: [*]const f32 = @ptrFromInt(a);
|
||||
const vb: [*]const f32 = @ptrFromInt(b);
|
||||
if (vb[0] <= va[0] and vb[1] <= va[1] and vb[2] <= va[2]) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// --- 0x749280: calculateSinCos ---
|
||||
pub fn si_calculateSinCos(angle_bits: u32, out_sin: u32, out_cos: u32) callconv(SC) void {
|
||||
const angle: f32 = @bitCast(angle_bits);
|
||||
const sp: *f32 = @ptrFromInt(out_sin);
|
||||
const cp: *f32 = @ptrFromInt(out_cos);
|
||||
sp.* = @sin(angle);
|
||||
cp.* = @cos(angle);
|
||||
}
|
||||
|
||||
// --- 0x7BE5B0: createZRotMat3x3 ---
|
||||
pub fn si_createZRotMat3x3(out: u32, angle_bits: u32) callconv(TC) u32 {
|
||||
const m: [*]f32 = @ptrFromInt(out);
|
||||
const angle: f32 = @bitCast(angle_bits);
|
||||
const c = @cos(angle); const s = @sin(angle);
|
||||
m[0] = c; m[1] = s; m[2] = 0;
|
||||
m[3] = -s; m[4] = c; m[5] = 0;
|
||||
m[6] = 0; m[7] = 0; m[8] = 1;
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- 0x7BCEF0: transposeMat4x4 ---
|
||||
// Naked thiscall: ECX=src, [ESP+4]=dst. RET 4. Original: 156 bytes.
|
||||
// SSE unpacklo/unpackhi transpose: 4 loads + 4 shuffles + 4 stores.
|
||||
pub fn si_transposeMat4x4() callconv(.naked) void {
|
||||
asm volatile (
|
||||
\\mov 4(%%esp), %%eax
|
||||
// Load 4 rows from src (ECX)
|
||||
\\vmovups (%%ecx), %%xmm0
|
||||
\\vmovups 16(%%ecx), %%xmm1
|
||||
\\vmovups 32(%%ecx), %%xmm2
|
||||
\\vmovups 48(%%ecx), %%xmm3
|
||||
// Transpose via unpacklo/unpackhi
|
||||
\\vunpcklps %%xmm1, %%xmm0, %%xmm4
|
||||
\\vunpckhps %%xmm1, %%xmm0, %%xmm5
|
||||
\\vunpcklps %%xmm3, %%xmm2, %%xmm6
|
||||
\\vunpckhps %%xmm3, %%xmm2, %%xmm7
|
||||
// Combine into final columns
|
||||
\\vmovlhps %%xmm6, %%xmm4, %%xmm0
|
||||
\\vmovhlps %%xmm4, %%xmm6, %%xmm1
|
||||
\\vmovlhps %%xmm7, %%xmm5, %%xmm2
|
||||
\\vmovhlps %%xmm5, %%xmm7, %%xmm3
|
||||
// Store to dst (EAX)
|
||||
\\vmovups %%xmm0, (%%eax)
|
||||
\\vmovups %%xmm1, 16(%%eax)
|
||||
\\vmovups %%xmm2, 32(%%eax)
|
||||
\\vmovups %%xmm3, 48(%%eax)
|
||||
// Return src in EAX
|
||||
\\mov %%ecx, %%eax
|
||||
\\ret $4
|
||||
);
|
||||
}
|
||||
|
||||
// --- 0x7BB420: mulMat3x4InPlace ---
|
||||
// this = this * matB. V4 columns loaded upfront, write directly back (no tmp needed).
|
||||
pub fn si_mulMat3x4InPlace(mat_a: u32, mat_b: u32) callconv(TC) u32 {
|
||||
const a: [*]f32 = @ptrFromInt(mat_a);
|
||||
const b: [*]const f32 = @ptrFromInt(mat_b);
|
||||
|
||||
// Load everything into locals to eliminate aliasing
|
||||
const ac0 = V4{ a[0], a[3], a[6], a[9] };
|
||||
const ac1 = V4{ a[1], a[4], a[7], a[10] };
|
||||
const ac2 = V4{ a[2], a[5], a[8], a[11] };
|
||||
const b_local: [12]f32 = .{ b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11] };
|
||||
|
||||
// Rotation: write directly back to a
|
||||
inline for (0..3) |row| {
|
||||
const br0: V4 = @splat(b_local[row * 3]);
|
||||
const br1: V4 = @splat(b_local[row * 3 + 1]);
|
||||
const br2: V4 = @splat(b_local[row * 3 + 2]);
|
||||
const result = @mulAdd(V4, ac2, br2, @mulAdd(V4, ac1, br1, ac0 * br0));
|
||||
a[row * 3] = result[0];
|
||||
a[row * 3 + 1] = result[1];
|
||||
a[row * 3 + 2] = result[2];
|
||||
}
|
||||
|
||||
// Translation
|
||||
inline for (0..3) |col| {
|
||||
a[9 + col] = @mulAdd(f32, ac2[3], b_local[col + 6], @mulAdd(f32, ac1[3], b_local[col + 3], @mulAdd(f32, ac0[3], b_local[col], b_local[9 + col])));
|
||||
}
|
||||
|
||||
return mat_a;
|
||||
}
|
||||
|
||||
// --- 0x6720F0: normalizeVec3InPlace ---
|
||||
// sqrt + reciprocal. 14cy (2.2x). rsqrt+NR tested at 15cy — no gain, compiler's
|
||||
// vsqrtss+vdivss pipeline is already optimal for scalar inverse sqrt.
|
||||
pub fn si_normalizeVec3InPlace(vec: u32) callconv(TC) void {
|
||||
const v: [*]f32 = @ptrFromInt(vec);
|
||||
const len = @sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
|
||||
if (len > 1.0e-20) {
|
||||
const inv = 1.0 / len;
|
||||
v[0] *= inv;
|
||||
v[1] *= inv;
|
||||
v[2] *= inv;
|
||||
}
|
||||
}
|
||||
|
||||
// --- 0x71BC70: addVec3ToAccumulator (136K/7.5s) ---
|
||||
// thiscall(ECX=this, stack=vec). Scale is a global at 0x81207C, NOT a parameter.
|
||||
pub fn si_addVec3ToAccumulator(this: u32, vec: u32) callconv(TC) void {
|
||||
const obj: [*]f32 = @ptrFromInt(this);
|
||||
const v: [*]const f32 = @ptrFromInt(vec);
|
||||
const scale: f32 = @as(*const f32, @ptrFromInt(0x81207C)).*;
|
||||
obj[21] += v[0];
|
||||
obj[22] += v[1];
|
||||
obj[23] += v[2];
|
||||
obj[33] = @mulAdd(f32, v[0], scale, obj[33]);
|
||||
obj[42] = @mulAdd(f32, v[1], scale, obj[42]);
|
||||
obj[51] = @mulAdd(f32, v[2], scale, obj[51]);
|
||||
}
|
||||
|
||||
// --- 0x71BF60: addToColorAccumulator (10K/7.5s) ---
|
||||
// Naked thiscall: ECX=this, [ESP+4]=color_ptr. RET 4. Original: 34 bytes.
|
||||
// 3 SSE adds at this+0x6C from color[0..2].
|
||||
pub fn si_addToColorAccumulator() callconv(.naked) void {
|
||||
asm volatile (
|
||||
\\mov 4(%%esp), %%eax
|
||||
\\vmovss (%%eax), %%xmm0
|
||||
\\vaddss 0x6C(%%ecx), %%xmm0, %%xmm0
|
||||
\\vmovss %%xmm0, 0x6C(%%ecx)
|
||||
\\vmovss 4(%%eax), %%xmm0
|
||||
\\vaddss 0x70(%%ecx), %%xmm0, %%xmm0
|
||||
\\vmovss %%xmm0, 0x70(%%ecx)
|
||||
\\vmovss 8(%%eax), %%xmm0
|
||||
\\vaddss 0x74(%%ecx), %%xmm0, %%xmm0
|
||||
\\vmovss %%xmm0, 0x74(%%ecx)
|
||||
\\ret $4
|
||||
);
|
||||
}
|
||||
|
||||
// --- 0x7B7A80: packParticleColor (2K/7.5s) ---
|
||||
// V4 multiply + clamp, then packed round+convert via @Vector(4, i32) for all channels at once.
|
||||
pub fn si_packParticleColor(obj: u32, r_bits: u32, g_bits: u32, b_bits: u32) callconv(TC) void {
|
||||
const base: [*]u8 = @ptrFromInt(obj);
|
||||
const out: *align(1) u32 = @ptrCast(base + 0x12C);
|
||||
const alpha = base[0x12F];
|
||||
const rgb = V4{ @bitCast(r_bits), @bitCast(g_bits), @bitCast(b_bits), 0 } * @as(V4, @splat(@as(f32, 255.0)));
|
||||
const clamped = @min(@max(rgb, @as(V4, @splat(@as(f32, 0.0)))), @as(V4, @splat(@as(f32, 255.0))));
|
||||
// Packed round + convert: single roundps + cvtps2dq (SSE4.1)
|
||||
const rounded = @round(clamped);
|
||||
const ints: @Vector(4, i32) = @intFromFloat(rounded);
|
||||
out.* = @as(u32, alpha) << 24 | @as(u32, @intCast(ints[0])) << 16 | @as(u32, @intCast(ints[1])) << 8 | @as(u32, @intCast(ints[2]));
|
||||
}
|
||||
|
||||
// --- 0x7B7B10: setParticleAlpha (2K/7.5s) ---
|
||||
// Naked fastcall: ECX=obj, [ESP+4]=alpha_bits. RET 4.
|
||||
// Clamp alpha*255 to [0,255], write byte to obj+0x12F.
|
||||
pub fn si_setParticleAlpha() callconv(.naked) void {
|
||||
asm volatile (
|
||||
\\vmovss 4(%%esp), %%xmm0
|
||||
\\mov $0x437F0000, %%eax
|
||||
\\vmovd %%eax, %%xmm1
|
||||
\\vmulss %%xmm1, %%xmm0, %%xmm0
|
||||
\\vxorps %%xmm2, %%xmm2, %%xmm2
|
||||
\\vmaxss %%xmm2, %%xmm0, %%xmm0
|
||||
\\vminss %%xmm1, %%xmm0, %%xmm0
|
||||
\\vcvtss2si %%xmm0, %%eax
|
||||
\\mov %%al, 0x12F(%%ecx)
|
||||
\\ret $4
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
// --- 0x40A2B0: __ftol ---
|
||||
// Drop-in binary replacement. Input: ST(0). Output: EAX:EDX (i64).
|
||||
// SSE3 FISTTP: truncate directly from x87 (9 bytes, replaces 39-byte original)
|
||||
pub fn si_ftol() callconv(.naked) void {
|
||||
asm volatile (
|
||||
\\sub $8, %%esp
|
||||
\\fisttpll (%%esp)
|
||||
\\pop %%eax
|
||||
\\pop %%edx
|
||||
\\ret
|
||||
);
|
||||
}
|
||||
|
||||
// --- 0x602630: vec3Dot (31K/7.5s) ---
|
||||
// __fastcall(ECX=a, EDX=b), returns f64 via ST(0).
|
||||
pub fn si_vec3Dot(a: u32, b: u32) callconv(FC) f64 {
|
||||
const va: [*]const f32 = @ptrFromInt(a);
|
||||
const vb: [*]const f32 = @ptrFromInt(b);
|
||||
return @floatCast(@mulAdd(f32, va[2], vb[2], @mulAdd(f32, va[1], vb[1], va[0] * vb[0])));
|
||||
}
|
||||
|
||||
// --- 0x686820: translateBoundingVol ---
|
||||
// @mulAdd for plane distances. Scalar corner adds (stride 3 — V4 unaligned tested, slower).
|
||||
pub fn si_translateBoundingVol(this: u32, offset: u32) callconv(TC) void {
|
||||
const obj: [*]f32 = @ptrFromInt(this);
|
||||
const off: [*]const f32 = @ptrFromInt(offset);
|
||||
const dx = off[0]; const dy = off[1]; const dz = off[2];
|
||||
inline for (0..8) |i| {
|
||||
const base = 24 + i * 3;
|
||||
obj[base] += dx;
|
||||
obj[base + 1] += dy;
|
||||
obj[base + 2] += dz;
|
||||
}
|
||||
inline for (0..6) |i| {
|
||||
const base = i * 4;
|
||||
obj[base + 3] -= @mulAdd(f32, obj[base + 2], dz, @mulAdd(f32, obj[base + 1], dy, obj[base] * dx));
|
||||
}
|
||||
obj[48] += dx; obj[49] += dy; obj[50] += dz;
|
||||
obj[51] += dx; obj[52] += dy; obj[53] += dz;
|
||||
}
|
||||
|
||||
// --- 0x686000: FrustumCullBoundingBox ---
|
||||
// Transforms bbox through view-proj matrix, perspective divides, projects to 320-column
|
||||
// occlusion buffer. Returns 0 (culled) / 2 (visible).
|
||||
// Original: 380 bytes, 2 calls to mat*vec3 (0x7BCA80), x87 perspective divide, x87 column scan.
|
||||
// SSE: inline V4 mat*vec3, SSE perspective divide, 4-wide column scan.
|
||||
// __fastcall(bbox_ECX, flags_EDX, radius_stack), RET 0x4
|
||||
pub fn si_frustumCullBBox(bbox: u32, flags: u32, radius_bits: u32) callconv(FC) u32 {
|
||||
// Early out: global occlusion flag bit 5
|
||||
if ((@as(*const u8, @ptrFromInt(0xC7B2A4)).* & 0x20) == 0) return 0;
|
||||
|
||||
// Early out: radius too small
|
||||
const radius: f32 = @bitCast(radius_bits);
|
||||
const epsilon: f32 = @bitCast(@as(*const u32, @ptrFromInt(0x8029D4)).*);
|
||||
if (@abs(radius) < epsilon) return 0;
|
||||
|
||||
// Early out: global value must be in valid range [const1, const2]
|
||||
const global_val: f32 = @as(*align(1) const f32, @ptrFromInt(0xC7CFF4)).*;
|
||||
if (global_val < @as(*align(1) const f32, @ptrFromInt(0x8101AC)).*) return 0;
|
||||
if (global_val > @as(*align(1) const f32, @ptrFromInt(0x804588)).*) return 0;
|
||||
|
||||
// Transform center through view-proj matrix (column-major 4x4 at 0xC7B700)
|
||||
// Inlined 0x7BCA80: result = col0*v.x + col1*v.y + col2*v.z + col3
|
||||
const bp: [*]const f32 = @ptrFromInt(bbox);
|
||||
const vx: V4 = @splat(bp[0]);
|
||||
const vy: V4 = @splat(bp[1]);
|
||||
const vz: V4 = @splat(bp[2]);
|
||||
|
||||
const m1: u32 = 0xC7B700;
|
||||
const center = @mulAdd(V4, vz, loadV4(m1 + 32), @mulAdd(V4, vy, loadV4(m1 + 16), @mulAdd(V4, vx, loadV4(m1), loadV4(m1 + 48))));
|
||||
|
||||
// Transform extent {radius, radius, 0} through matrix at 0xC7D280
|
||||
const rv: V4 = @splat(radius);
|
||||
const m2: u32 = 0xC7D280;
|
||||
// z=0, so skip col2 term
|
||||
const extent = @mulAdd(V4, rv, loadV4(m2 + 16), @mulAdd(V4, rv, loadV4(m2), loadV4(m2 + 48)));
|
||||
|
||||
// Behind-camera check (unless flags & 8)
|
||||
if ((flags & 0x8) == 0) {
|
||||
if (center[2] < @as(*align(1) const f32, @ptrFromInt(0x80FED4)).*) return 0;
|
||||
}
|
||||
|
||||
// Perspective divide: inv_w = K * rcpss(center.z) with Newton-Raphson refinement.
|
||||
// vrcpss + NR: ~5 cycles vs vdivss ~14 cycles. Column indices only need ~1px accuracy.
|
||||
const K: f32 = @as(*align(1) const f32, @ptrFromInt(0x7FF9D8)).*;
|
||||
const cz = center[2];
|
||||
const inv_w = K * fastRecip(cz);
|
||||
const cx = center[0] * inv_w;
|
||||
const ex = extent[0] * inv_w;
|
||||
const ey = extent[1] * inv_w;
|
||||
const depth = @mulAdd(f32, center[1], inv_w, ex);
|
||||
|
||||
// Column projection: convert to 320-column indices
|
||||
const col_scale: f32 = @as(*align(1) const f32, @ptrFromInt(0x810170)).*;
|
||||
const col_offset: f32 = @as(*align(1) const f32, @ptrFromInt(0x86861C)).*;
|
||||
|
||||
// cvtss2si: round-to-nearest in one instruction (MXCSR default mode).
|
||||
// Replaces @round + @intFromFloat which generates vroundss + sign handling (~6 insns).
|
||||
var left_col: i32 = cvtss2si(@mulAdd(f32, cx - ey, col_scale, -col_offset));
|
||||
left_col += 0xA0;
|
||||
var right_col: i32 = cvtss2si(@mulAdd(f32, ey + cx, col_scale, -col_offset));
|
||||
right_col += 0xA1;
|
||||
|
||||
// Bounds check — off-screen culling
|
||||
if (left_col >= 0x140) return 0; // fully right of screen (320)
|
||||
if (right_col < 0) return 0; // fully left of screen
|
||||
if (left_col < 0) left_col = 0;
|
||||
if (right_col >= 0x140) right_col = 0x13F; // clamp to 319
|
||||
if (left_col > right_col) return 2; // degenerate → visible
|
||||
|
||||
// Horizon buffer scan: 320 floats at 0xC7B750
|
||||
// If any column's horizon value < depth → culled (return 0)
|
||||
// SSE: test 4 columns at once
|
||||
const horizon_base: u32 = 0xC7B750;
|
||||
var col: u32 = @intCast(left_col);
|
||||
const end: u32 = @intCast(right_col);
|
||||
const depth_v: V4 = @splat(depth);
|
||||
|
||||
// 4-wide scan
|
||||
while (col + 3 <= end) {
|
||||
const h = loadV4(horizon_base + col * 4);
|
||||
const lt_bits: u4 = @bitCast(h < depth_v);
|
||||
if (lt_bits != 0) return 0;
|
||||
col += 4;
|
||||
}
|
||||
// Scalar remainder
|
||||
while (col <= end) {
|
||||
if (@as(*align(1) const f32, @ptrFromInt(horizon_base + col * 4)).* < depth) return 0;
|
||||
col += 1;
|
||||
}
|
||||
|
||||
return 2; // visible — survived all columns
|
||||
}
|
||||
|
||||
// --- 0x6ABC40: processLinkedListCollision ---
|
||||
// Walks intrusive linked list, per-node AABB overlap test, calls addGeometryToBuffer on hit.
|
||||
// Original: 329 bytes, 6 x87 FCOMP/FNSTSW comparisons per node.
|
||||
// SSE: 2 V4 loads + 2 CMPPS + AND + MOVMSK replaces the 6 scalar comparisons.
|
||||
//
|
||||
// __fastcall(listHead_ECX, queryBox_EDX, resultBuf_stack, flags_stack), RET 0x8
|
||||
// addGeometryToBuffer at 0x6ABD90: __fastcall(queryBox_ECX, nodeData_EDX, resultBuf_stack), RET 0x4
|
||||
// Visited sentinel: *(u32*)0xC89F20
|
||||
pub fn si_processLinkedListCollision(list_head: u32, query_box: u32, result_buf: u32, flags: u32) callconv(FC) u32 {
|
||||
if ((flags & 0xF0000F) == 0) return 1;
|
||||
|
||||
const addGeometryToBuffer: *const fn (u32, u32, u32) callconv(FC) void = @ptrFromInt(0x6ABD90);
|
||||
|
||||
// Load query box min/max as V4 for SSE AABB test
|
||||
// queryBox layout: min(+0,+4,+8), max(+0xC,+0x10,+0x14)
|
||||
const q_min = loadV4(query_box); // {qmin.x, qmin.y, qmin.z, <garbage>}
|
||||
const q_max = loadV4(query_box + 0x0C); // {qmax.x, qmax.y, qmax.z, <garbage>}
|
||||
|
||||
const sentinel = @as(*const u32, @ptrFromInt(0xC89F20)).*;
|
||||
const link_offset = @as(*const u32, @ptrFromInt(list_head)).*;
|
||||
|
||||
// First node: listHead[2] (offset +8)
|
||||
var node: u32 = @as(*const u32, @ptrFromInt(list_head + 8)).*;
|
||||
|
||||
// Linked list tag bit: bit 0 set = end sentinel
|
||||
if (node & 1 != 0 or node == 0) return 1;
|
||||
|
||||
while (node & 1 == 0 and node != 0) {
|
||||
const node_data = @as(*const u32, @ptrFromInt(node + 4)).*;
|
||||
const prev_node = node;
|
||||
|
||||
// Skip: flags bit 0x100 set
|
||||
const node_flags = @as(*const u16, @ptrFromInt(node_data + 0x0C)).*;
|
||||
if ((node_flags & 0x100) == 0) {
|
||||
// Skip: already visited or null
|
||||
const visited = @as(*const u32, @ptrFromInt(node_data + 0x8C)).*;
|
||||
const active = @as(*const u32, @ptrFromInt(node_data + 0x88)).*;
|
||||
if (visited != sentinel and active != 0) {
|
||||
// Type discriminator: pick flag mask
|
||||
const type_a = @as(*const u32, @ptrFromInt(node_data + 0x180)).*;
|
||||
const type_b = @as(*const u32, @ptrFromInt(node_data + 0x184)).*;
|
||||
const mask = if ((type_a | type_b) != 0) flags & 0xF00000 else flags & 0xF;
|
||||
|
||||
if (mask != 0) {
|
||||
// Bit 7 of flags byte: if clear, abort with 0
|
||||
if (@as(i8, @bitCast(@as(u8, @truncate(node_flags)))) >= 0) return 0;
|
||||
|
||||
// --- SSE AABB overlap test ---
|
||||
// node AABB at node_data+0x14C: min(3 floats), max(3 floats)
|
||||
const n_min = loadV4(node_data + 0x14C); // {nmin.x, nmin.y, nmin.z, <nmax.x>}
|
||||
const n_max = loadV4(node_data + 0x158); // {nmax.x, nmax.y, nmax.z, <garbage>}
|
||||
|
||||
// Overlap: nodeMin < queryMax AND queryMin <= nodeMax
|
||||
// Compare lane-wise, check low 3 bits of mask
|
||||
const lt_mask = n_min < q_max;
|
||||
const le_mask = q_min <= n_max;
|
||||
const lt_bits: u4 = @bitCast(lt_mask);
|
||||
const le_bits: u4 = @bitCast(le_mask);
|
||||
const bits = lt_bits & le_bits;
|
||||
|
||||
if ((bits & 0x7) == 0x7) {
|
||||
addGeometryToBuffer(query_box, node_data, result_buf);
|
||||
}
|
||||
|
||||
// Mark visited
|
||||
@as(*u32, @ptrFromInt(node_data + 0x8C)).* = sentinel;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Advance: next = *(node + link_offset + 4)
|
||||
// Original: MOV ECX,[EAX + EDX*1 + 4] where EAX=*listHead, EDX=node
|
||||
node = @as(*const u32, @ptrFromInt(link_offset + prev_node + 4)).*;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
//! timer_fix -- TSC calibration + OS timer tweaks (ported from VanillaFixes)
|
||||
//!
|
||||
//! The vanilla client poorly calibrates its RDTSC frequency, causing animation
|
||||
//! stutter. We recalibrate using QueryPerformanceFrequency as reference and
|
||||
//! patch the game's timer globals. A/B tested: BASELINE restores original
|
||||
//! values, CUSTOM applies calibrated values.
|
||||
//!
|
||||
//! Timer globals (1.12.1 build 5875):
|
||||
//! pUseTSC: 0x00884c80 (BOOL)
|
||||
//! pTimerTicksPerSecond: 0x008332c0 (u64)
|
||||
//! pTimerToMilliseconds: 0x008332c8 (f64)
|
||||
//! pTimerOffset: 0x00884c88 (f64)
|
||||
|
||||
const hook = @import("zhook");
|
||||
|
||||
const ADDR_USE_TSC: usize = 0x00884c80;
|
||||
const ADDR_TICKS_PER_SEC: usize = 0x008332c0;
|
||||
const ADDR_TO_MS: usize = 0x008332c8;
|
||||
const ADDR_OFFSET: usize = 0x00884c88;
|
||||
|
||||
const WINAPI = @import("std").builtin.CallingConvention.winapi;
|
||||
const LARGE_INTEGER = extern struct { QuadPart: i64 };
|
||||
|
||||
extern "kernel32" fn QueryPerformanceCounter(lpPerformanceCount: *LARGE_INTEGER) callconv(WINAPI) i32;
|
||||
extern "kernel32" fn QueryPerformanceFrequency(lpFrequency: *LARGE_INTEGER) callconv(WINAPI) i32;
|
||||
extern "kernel32" fn GetTickCount() callconv(WINAPI) u32;
|
||||
extern "kernel32" fn GetCurrentThread() callconv(WINAPI) usize;
|
||||
extern "kernel32" fn SetThreadPriority(hThread: usize, nPriority: i32) callconv(WINAPI) i32;
|
||||
extern "kernel32" fn GetCurrentProcessorNumber() callconv(WINAPI) u32;
|
||||
extern "kernel32" fn SetThreadAffinityMask(hThread: usize, dwThreadAffinityMask: usize) callconv(WINAPI) usize;
|
||||
extern "kernel32" fn Sleep(dwMilliseconds: u32) callconv(WINAPI) void;
|
||||
|
||||
// Windows API for OS timer tweaks
|
||||
extern "kernel32" fn GetModuleHandleA(lpModuleName: [*:0]const u8) callconv(WINAPI) ?*anyopaque;
|
||||
extern "kernel32" fn GetProcAddress(hModule: *anyopaque, lpProcName: [*:0]const u8) callconv(WINAPI) ?*anyopaque;
|
||||
extern "kernel32" fn GetCurrentProcess() callconv(WINAPI) usize;
|
||||
|
||||
fn increaseTimerResolution() void {
|
||||
const ntdll = GetModuleHandleA("ntdll") orelse return;
|
||||
|
||||
const NtQueryFn = *const fn (*u32, *u32, *u32) callconv(WINAPI) i32;
|
||||
const NtSetFn = *const fn (u32, i32, *u32) callconv(WINAPI) i32;
|
||||
|
||||
const query_ptr = GetProcAddress(ntdll, "NtQueryTimerResolution");
|
||||
const set_ptr = GetProcAddress(ntdll, "NtSetTimerResolution");
|
||||
|
||||
if (query_ptr != null and set_ptr != null) {
|
||||
const query: NtQueryFn = @ptrCast(query_ptr.?);
|
||||
const set: NtSetFn = @ptrCast(set_ptr.?);
|
||||
|
||||
var min_res: u32 = 0;
|
||||
var max_res: u32 = 0;
|
||||
var cur_res: u32 = 0;
|
||||
_ = query(&min_res, &max_res, &cur_res);
|
||||
|
||||
// Request 0.5ms (5000 * 100ns) or hardware max, whichever is larger
|
||||
const desired = @max(max_res, 5000);
|
||||
var actual: u32 = 0;
|
||||
_ = set(desired, 1, &actual); // 1 = TRUE (enable)
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback: timeBeginPeriod(1) via winmm
|
||||
const winmm = GetModuleHandleA("winmm") orelse return;
|
||||
const TbpFn = *const fn (u32) callconv(WINAPI) u32;
|
||||
const tbp_ptr = GetProcAddress(winmm, "timeBeginPeriod") orelse return;
|
||||
const tbp: TbpFn = @ptrCast(tbp_ptr);
|
||||
_ = tbp(1);
|
||||
}
|
||||
|
||||
fn disablePowerThrottling() void {
|
||||
const kernel32 = GetModuleHandleA("kernel32") orelse return;
|
||||
const SpiPtr = GetProcAddress(kernel32, "SetProcessInformation") orelse return;
|
||||
|
||||
// ProcessPowerThrottling = 4
|
||||
const PowerThrottlingState = extern struct {
|
||||
Version: u32,
|
||||
ControlMask: u32,
|
||||
StateMask: u32,
|
||||
};
|
||||
const SetProcessInfoFn = *const fn (usize, u32, *PowerThrottlingState, u32) callconv(WINAPI) i32;
|
||||
const setProcessInfo: SetProcessInfoFn = @ptrCast(SpiPtr);
|
||||
const process = GetCurrentProcess();
|
||||
|
||||
// PROCESS_POWER_THROTTLING_IGNORE_TIMER_RESOLUTION = 2
|
||||
var state1 = PowerThrottlingState{ .Version = 1, .ControlMask = 2, .StateMask = 0 };
|
||||
_ = setProcessInfo(process, 4, &state1, @sizeOf(PowerThrottlingState));
|
||||
|
||||
// PROCESS_POWER_THROTTLING_EXECUTION_SPEED = 1
|
||||
var state2 = PowerThrottlingState{ .Version = 1, .ControlMask = 1, .StateMask = 0 };
|
||||
_ = setProcessInfo(process, 4, &state2, @sizeOf(PowerThrottlingState));
|
||||
}
|
||||
|
||||
// Saved original values (before calibration) for A/B testing
|
||||
var orig_ticks_per_sec: u64 = 0;
|
||||
var orig_to_ms: f64 = 0;
|
||||
var orig_offset: f64 = 0;
|
||||
|
||||
// Calibrated values
|
||||
var cal_ticks_per_sec: u64 = 0;
|
||||
var cal_to_ms: f64 = 0;
|
||||
var cal_offset: f64 = 0;
|
||||
|
||||
var is_calibrated: bool = false;
|
||||
|
||||
inline fn rdtsc() u64 {
|
||||
var lo: u32 = undefined;
|
||||
var hi: u32 = undefined;
|
||||
asm volatile ("rdtsc"
|
||||
: [lo] "={eax}" (lo),
|
||||
[hi] "={edx}" (hi),
|
||||
);
|
||||
return @as(u64, hi) << 32 | lo;
|
||||
}
|
||||
|
||||
fn timeSample() struct { qpc: i64, tsc: u64 } {
|
||||
Sleep(0); // yield to get fresh timeslice
|
||||
var qpc: LARGE_INTEGER = .{ .QuadPart = 0 };
|
||||
_ = QueryPerformanceCounter(&qpc);
|
||||
const tsc = rdtsc();
|
||||
asm volatile ("lfence" ::: "memory");
|
||||
return .{ .qpc = qpc.QuadPart, .tsc = tsc };
|
||||
}
|
||||
|
||||
fn calibrateTSC() u64 {
|
||||
const thread = GetCurrentThread();
|
||||
_ = SetThreadPriority(thread, 2); // THREAD_PRIORITY_HIGHEST
|
||||
const core: u5 = @truncate(GetCurrentProcessorNumber());
|
||||
_ = SetThreadAffinityMask(thread, @as(usize, 1) << core);
|
||||
|
||||
const baseline = timeSample();
|
||||
Sleep(500);
|
||||
const after = timeSample();
|
||||
|
||||
_ = SetThreadPriority(thread, 0); // THREAD_PRIORITY_NORMAL
|
||||
_ = SetThreadAffinityMask(thread, 0); // all cores
|
||||
|
||||
var freq: LARGE_INTEGER = .{ .QuadPart = 0 };
|
||||
_ = QueryPerformanceFrequency(&freq);
|
||||
|
||||
const qpc_delta: f64 = @floatFromInt(after.qpc - baseline.qpc);
|
||||
const tsc_delta: f64 = @floatFromInt(after.tsc - baseline.tsc);
|
||||
const elapsed_sec = qpc_delta / @as(f64, @floatFromInt(freq.QuadPart));
|
||||
|
||||
return @intFromFloat(@round(tsc_delta / elapsed_sec));
|
||||
}
|
||||
|
||||
fn writeTimerValues(ticks: u64, to_ms: f64, offset: f64) void {
|
||||
const ticks_bytes = @as([8]u8, @bitCast(ticks));
|
||||
hook.writeProtected(ADDR_TICKS_PER_SEC, &ticks_bytes);
|
||||
const ms_bytes = @as([8]u8, @bitCast(to_ms));
|
||||
hook.writeProtected(ADDR_TO_MS, &ms_bytes);
|
||||
const off_bytes = @as([8]u8, @bitCast(offset));
|
||||
hook.writeProtected(ADDR_OFFSET, &off_bytes);
|
||||
}
|
||||
|
||||
/// Calibrate and save both original and calibrated values.
|
||||
/// Called once from transform44.installHooks().
|
||||
pub fn init() void {
|
||||
// OS-level tweaks (independent of TSC calibration)
|
||||
increaseTimerResolution();
|
||||
disablePowerThrottling();
|
||||
|
||||
const tsc_was_enabled = hook.readMem(u32, ADDR_USE_TSC) != 0;
|
||||
|
||||
// Save original values (if TSC was already enabled)
|
||||
if (tsc_was_enabled) {
|
||||
orig_ticks_per_sec = hook.readMem(u64, ADDR_TICKS_PER_SEC);
|
||||
const orig_ms_bytes = hook.readMem(u64, ADDR_TO_MS);
|
||||
orig_to_ms = @bitCast(orig_ms_bytes);
|
||||
const orig_off_bytes = hook.readMem(u64, ADDR_OFFSET);
|
||||
orig_offset = @bitCast(orig_off_bytes);
|
||||
} else {
|
||||
// TSC not in use -- game is using GetTickCount (1000 ticks/sec).
|
||||
// Save those as "original" for A/B comparison.
|
||||
orig_ticks_per_sec = 1000;
|
||||
orig_to_ms = 1.0;
|
||||
orig_offset = 0;
|
||||
}
|
||||
|
||||
// Calibrate
|
||||
cal_ticks_per_sec = calibrateTSC();
|
||||
cal_to_ms = 1000.0 / @as(f64, @floatFromInt(cal_ticks_per_sec));
|
||||
|
||||
// Align TSC epoch with GetTickCount
|
||||
const gtc: f64 = @floatFromInt(GetTickCount());
|
||||
const tsc_ms: f64 = @as(f64, @floatFromInt(rdtsc())) * cal_to_ms;
|
||||
cal_offset = gtc - tsc_ms;
|
||||
|
||||
// Check if already well-calibrated (within 0.1%)
|
||||
if (tsc_was_enabled) {
|
||||
const orig_f: f64 = @floatFromInt(orig_ticks_per_sec);
|
||||
const cal_f: f64 = @floatFromInt(cal_ticks_per_sec);
|
||||
const diff_pct = @abs(cal_f - orig_f) / cal_f * 100.0;
|
||||
|
||||
if (diff_pct < 0.1) {
|
||||
// Already calibrated (VanillaFixes or similar), keep original
|
||||
is_calibrated = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
is_calibrated = true;
|
||||
// Enable TSC mode if it wasn't already
|
||||
if (!tsc_was_enabled) {
|
||||
const one = @as([4]u8, @bitCast(@as(u32, 1)));
|
||||
hook.writeProtected(ADDR_USE_TSC, &one);
|
||||
}
|
||||
// Apply calibrated values
|
||||
writeTimerValues(cal_ticks_per_sec, cal_to_ms, cal_offset);
|
||||
}
|
||||
|
||||
/// Switch to calibrated (CUSTOM) timer values.
|
||||
pub fn applyCalibrated() void {
|
||||
if (!is_calibrated) return;
|
||||
const one = @as([4]u8, @bitCast(@as(u32, 1)));
|
||||
hook.writeProtected(ADDR_USE_TSC, &one);
|
||||
writeTimerValues(cal_ticks_per_sec, cal_to_ms, cal_offset);
|
||||
}
|
||||
|
||||
/// Restore original (BASELINE) timer values.
|
||||
pub fn applyOriginal() void {
|
||||
if (!is_calibrated) return;
|
||||
// If TSC was originally disabled, restore that state
|
||||
if (orig_ticks_per_sec == 1000) {
|
||||
const zero = @as([4]u8, @bitCast(@as(u32, 0)));
|
||||
hook.writeProtected(ADDR_USE_TSC, &zero);
|
||||
}
|
||||
writeTimerValues(orig_ticks_per_sec, orig_to_ms, orig_offset);
|
||||
}
|
||||
|
||||
pub const TimerInfo = struct {
|
||||
calibrated: bool,
|
||||
orig_freq: u64,
|
||||
cal_freq: u64,
|
||||
diff_pct_x10: u64, // tenths of a percent
|
||||
};
|
||||
|
||||
pub fn getInfo() TimerInfo {
|
||||
const orig_f: f64 = @floatFromInt(orig_ticks_per_sec);
|
||||
const cal_f: f64 = @floatFromInt(cal_ticks_per_sec);
|
||||
const diff = if (cal_f > 0) @abs(cal_f - orig_f) / cal_f * 1000.0 else 0;
|
||||
return .{
|
||||
.calibrated = is_calibrated,
|
||||
.orig_freq = orig_ticks_per_sec,
|
||||
.cal_freq = cal_ticks_per_sec,
|
||||
.diff_pct_x10 = @intFromFloat(diff),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
//! performance — production SSE replacements with zero profiling overhead.
|
||||
//!
|
||||
//! Consolidates all verified permanent optimizations from transform44 and silicon
|
||||
//! into a single clean module. No rdtsc, no A/B testing, no probe counters.
|
||||
//!
|
||||
//! Hooks:
|
||||
//! - transformMatrix4x4 (bone SSE, 0x714260)
|
||||
//! - RenderParticleSprites (particle SSE, 0x7B2A50)
|
||||
//! - GetOrCreateCharacterGlyph (glyph cache, 0x5CA2D0)
|
||||
//! - OnWorldUpdate (0x482EA0) — per-frame cache reset
|
||||
//!
|
||||
//! JMP patches (via silicon patch table):
|
||||
//! - All silicon_sse functions (frustumCull, processLinkedListCollision, ftol, etc.)
|
||||
//! are installed by the silicon module — not duplicated here.
|
||||
|
||||
const hook = @import("zhook");
|
||||
const logging = @import("../logging.zig");
|
||||
const mod_mutex = @import("../mutex.zig");
|
||||
const inflate_hook = @import("inflate_hook.zig");
|
||||
const timer_fix = @import("timer_fix.zig");
|
||||
const filecache = @import("filecache.zig");
|
||||
|
||||
pub const module_name: [*:0]const u8 = "weirdperformance";
|
||||
|
||||
// malloc/free for libdeflate provided by stubs/game_alloc.c (compiled into
|
||||
// the libdeflate static lib). This avoids exporting malloc/free from the DLL.
|
||||
|
||||
var g_mutex: ?*anyopaque = null;
|
||||
var g_is_hook_owner: bool = false;
|
||||
var log: logging.Logger = .{};
|
||||
|
||||
pub fn isActive() bool {
|
||||
return g_is_hook_owner;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// SSE functions (imported directly to avoid addObject SizeOfImage bloat)
|
||||
// =============================================================================
|
||||
|
||||
const bone_sse = @import("bone_sse.zig");
|
||||
const particle_sse = @import("particle_sse.zig");
|
||||
const clip_sse = @import("clip_sse.zig");
|
||||
const cull_sse = @import("cull_sse.zig");
|
||||
const silicon_sse = @import("silicon_sse.zig");
|
||||
|
||||
const transformImpl_SSE = bone_sse.transformImpl_SSE;
|
||||
const renderParticleSprites_SSE = particle_sse.renderParticleSprites_SSE;
|
||||
const resetParticleCache = particle_sse.resetParticleCache;
|
||||
|
||||
// =============================================================================
|
||||
// transformMatrix4x4 hook (0x714260)
|
||||
// =============================================================================
|
||||
|
||||
fn transformMatrix4x4_SSE(this: u32, mat1: u32, mat2: u32, mat3: u32, mat4: u32) callconv(.{ .x86_thiscall = .{} }) void {
|
||||
transformImpl_SSE(this, mat1, mat2, mat3, mat4);
|
||||
}
|
||||
|
||||
const TransformFn = fn (u32, u32, u32, u32, u32) callconv(.{ .x86_thiscall = .{} }) void;
|
||||
var transform_hook: hook.Detour(TransformFn) = .{};
|
||||
|
||||
fn transformDetour(this: u32, mat1: u32, mat2: u32, mat3: u32, mat4: u32) callconv(.{ .x86_thiscall = .{} }) void {
|
||||
transformMatrix4x4_SSE(this, mat1, mat2, mat3, mat4);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// RenderParticleSprites hook (0x7B2A50)
|
||||
// =============================================================================
|
||||
|
||||
const ParticleFn = fn (u32, u32, u32, u32) callconv(hook.cc.fastcall) ?*anyopaque;
|
||||
var particle_hook: hook.Detour(ParticleFn) = .{};
|
||||
|
||||
fn particleDetour(a: u32, _: u32, c: u32, d: u32) callconv(hook.cc.fastcall) ?*anyopaque {
|
||||
return @ptrFromInt(renderParticleSprites_SSE(a, c, d));
|
||||
}
|
||||
|
||||
|
||||
// =============================================================================
|
||||
// GetOrCreateCharacterGlyph cache (0x5CA2D0)
|
||||
// =============================================================================
|
||||
|
||||
const GLYPH_CACHE_SHIFT = 12;
|
||||
const GLYPH_CACHE_SIZE = 1 << GLYPH_CACHE_SHIFT;
|
||||
const GLYPH_CACHE_MASK = GLYPH_CACHE_SIZE - 1;
|
||||
|
||||
const GlyphCacheEntry = struct {
|
||||
font_ptr: u32 = 0,
|
||||
char_code: u32 = 0,
|
||||
param2: u32 = 0,
|
||||
scale_bits: u32 = 0, // font+0x188 scale factor (changes on resize)
|
||||
width_bits: u32 = 0,
|
||||
};
|
||||
|
||||
var glyph_cache: [GLYPH_CACHE_SIZE]GlyphCacheEntry = [_]GlyphCacheEntry{.{}} ** GLYPH_CACHE_SIZE;
|
||||
|
||||
const GlyphFn = fn (u32, u32, u32, u32) callconv(hook.cc.fastcall) ?*anyopaque;
|
||||
var glyph_hook: hook.Detour(GlyphFn) = .{};
|
||||
|
||||
fn glyphDetour(a: u32, b: u32, c: u32, d: u32) callconv(hook.cc.fastcall) ?*anyopaque {
|
||||
asm volatile ("" ::: .{ .esi = true, .edi = true, .ebx = true });
|
||||
|
||||
const scale_bits = hook.readMem(u32, a + 0x188);
|
||||
const hash = ((a ^ c *% 0x9E3779B9) ^ d ^ scale_bits) & GLYPH_CACHE_MASK;
|
||||
const entry = &glyph_cache[hash];
|
||||
|
||||
if (entry.font_ptr == a and entry.char_code == c and entry.param2 == d and entry.scale_bits == scale_bits) {
|
||||
asm volatile ("flds (%[p])"
|
||||
:
|
||||
: [p] "r" (&entry.width_bits),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const ret = glyph_hook.callOriginal(.{ a, b, c, d });
|
||||
|
||||
var width_bits: u32 = undefined;
|
||||
asm volatile ("fsts (%[p])"
|
||||
:
|
||||
: [p] "r" (&width_bits),
|
||||
);
|
||||
|
||||
entry.* = .{
|
||||
.font_ptr = a,
|
||||
.char_code = c,
|
||||
.param2 = d,
|
||||
.scale_bits = scale_bits,
|
||||
.width_bits = width_bits,
|
||||
};
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// FindObjectByGUID cache (0x464890) — direct-mapped, validate-on-hit
|
||||
// =============================================================================
|
||||
|
||||
const FindGuidFn = fn (u32, u32, u32, u32) callconv(hook.cc.fastcall) ?*anyopaque;
|
||||
var findguid_hook: hook.Detour(FindGuidFn) = .{};
|
||||
|
||||
const GUID_CACHE_BITS = 12;
|
||||
const GUID_CACHE_SIZE = 1 << GUID_CACHE_BITS;
|
||||
const GUID_CACHE_MASK = GUID_CACHE_SIZE - 1;
|
||||
const GuidCacheEntry = struct { guid_lo: u32 = 0, guid_hi: u32 = 0, result: u32 = 0 };
|
||||
var guid_cache: [GUID_CACHE_SIZE]GuidCacheEntry = [_]GuidCacheEntry{.{}} ** GUID_CACHE_SIZE;
|
||||
|
||||
// MoveObjectToDeletedList: stdcall(guidLow, guidHigh), RET 0x8
|
||||
// Same fastcall(4) Detour mapping as FindObjectByGUID
|
||||
const ObjDeleteFn = fn (u32, u32, u32, u32) callconv(hook.cc.fastcall) ?*anyopaque;
|
||||
var obj_delete_hook: hook.Detour(ObjDeleteFn) = .{};
|
||||
|
||||
fn objDeleteDetour(a: u32, b: u32, c: u32, d: u32) callconv(hook.cc.fastcall) ?*anyopaque {
|
||||
// Let original run first (it calls FindObjectByGUID internally, which re-caches).
|
||||
// Then evict, so the stale entry is removed after the original is done.
|
||||
const ret = obj_delete_hook.callOriginal(.{ a, b, c, d });
|
||||
const idx = (c ^ d) & GUID_CACHE_MASK;
|
||||
const entry = &guid_cache[idx];
|
||||
if (entry.guid_lo == c and entry.guid_hi == d) {
|
||||
entry.* = .{};
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
fn findguidDetour(a: u32, b: u32, c: u32, d: u32) callconv(hook.cc.fastcall) ?*anyopaque {
|
||||
// stdcall(2): c=guidLow, d=guidHigh (a,b unused fastcall reg args)
|
||||
const idx = (c ^ d) & GUID_CACHE_MASK;
|
||||
const entry = &guid_cache[idx];
|
||||
|
||||
// No pointer validation needed -- cache is flushed every frame
|
||||
if (entry.guid_lo == c and entry.guid_hi == d and entry.result != 0) {
|
||||
return @ptrFromInt(entry.result);
|
||||
}
|
||||
|
||||
const ret = findguid_hook.callOriginal(.{ a, b, c, d });
|
||||
const result = @intFromPtr(ret);
|
||||
if (result != 0) {
|
||||
entry.* = .{ .guid_lo = c, .guid_hi = d, .result = result };
|
||||
} else {
|
||||
// Object not found -- clear cache entry to prevent stale hits
|
||||
entry.* = .{};
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// DestroyObjectManager hook (0x467700) — flush GUID cache before teardown
|
||||
// =============================================================================
|
||||
|
||||
var destroy_objmgr_hook: hook.Detour(fn () callconv(hook.cc.stdcall) void) = .{};
|
||||
|
||||
fn destroyObjMgrDetour() callconv(hook.cc.stdcall) void {
|
||||
guid_cache = [_]GuidCacheEntry{.{}} ** GUID_CACHE_SIZE;
|
||||
destroy_objmgr_hook.callOriginal(.{});
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// OnWorldUpdate hook (0x482EA0) — per-frame cache reset
|
||||
// =============================================================================
|
||||
|
||||
const WorldUpdateFn = fn (u32) callconv(hook.cc.fastcall) void;
|
||||
var world_update_hook: hook.Detour(WorldUpdateFn) = .{};
|
||||
|
||||
var frame_counter: u32 = 0;
|
||||
|
||||
fn worldUpdateDetour(frame_count: u32) callconv(hook.cc.fastcall) void {
|
||||
resetParticleCache();
|
||||
world_update_hook.callOriginal(.{frame_count});
|
||||
}
|
||||
|
||||
|
||||
// =============================================================================
|
||||
// SSE JMP patches — binary patches at game function addresses
|
||||
// =============================================================================
|
||||
|
||||
// cull_sse.zig
|
||||
const performSpatialCulling = cull_sse.performSpatialCulling;
|
||||
const performCollisionDetectionSSE = cull_sse.performCollisionDetectionSSE;
|
||||
const rayTriIntersectIndexedInt = cull_sse.rayTriIntersectIndexedInt;
|
||||
|
||||
// silicon_sse.zig
|
||||
const sse = silicon_sse;
|
||||
|
||||
const PatchEntry = struct {
|
||||
target: u32,
|
||||
replacement: u32,
|
||||
name: [*:0]const u8,
|
||||
direct_size: u8 = 0,
|
||||
};
|
||||
|
||||
fn patchJmp(target: u32, replacement: u32) void {
|
||||
const rel = @as(i32, @bitCast(replacement -% target -% 5));
|
||||
var patch = [5]u8{ 0xE9, 0, 0, 0, 0 };
|
||||
@as(*align(1) i32, @ptrCast(patch[1..5])).* = rel;
|
||||
hook.writeProtected(target, &patch);
|
||||
}
|
||||
|
||||
fn patchDirect(target: u32, src: [*]const u8, size: usize) void {
|
||||
hook.writeProtected(target, src[0..size]);
|
||||
}
|
||||
|
||||
fn installPatches() u32 {
|
||||
const patches = [_]PatchEntry{
|
||||
.{ .target = 0x4549C0, .replacement = @intFromPtr(&sse.si_normalizeVec3), .name = "normalizeVec3" },
|
||||
.{ .target = 0x7BAE60, .replacement = @intFromPtr(&sse.si_mulMat3x4), .name = "mulMat3x4" },
|
||||
.{ .target = 0x7BDDB0, .replacement = @intFromPtr(&sse.si_rotateMatByQuat), .name = "rotateMatByQuat" },
|
||||
.{ .target = 0x7BB860, .replacement = @intFromPtr(&sse.si_createRotMat3x4), .name = "createRotMat3x4" },
|
||||
.{ .target = 0x686C20, .replacement = @intFromPtr(&sse.si_classifyPointFrustum), .name = "classifyPointFrustum" },
|
||||
.{ .target = 0x6DC5A0, .replacement = @intFromPtr(&sse.si_checkBoxLineIntersect), .name = "checkBoxLineIntersect" },
|
||||
.{ .target = 0x6869C0, .replacement = @intFromPtr(&sse.si_testOBBFrustum), .name = "testOBBFrustum" },
|
||||
.{ .target = 0x686B80, .replacement = @intFromPtr(&sse.si_testSphereFrustum), .name = "testSphereFrustum" },
|
||||
.{ .target = 0x7C0570, .replacement = @intFromPtr(&sse.si_quatSlerp), .name = "quatSlerp" },
|
||||
.{ .target = 0x749280, .replacement = @intFromPtr(&sse.si_calculateSinCos), .name = "calculateSinCos" },
|
||||
.{ .target = 0x7BE5B0, .replacement = @intFromPtr(&sse.si_createZRotMat3x3), .name = "createZRotMat3x3" },
|
||||
.{ .target = 0x7BCEF0, .replacement = @intFromPtr(&sse.si_transposeMat4x4), .name = "transposeMat4x4", .direct_size = 64 },
|
||||
.{ .target = 0x7BB420, .replacement = @intFromPtr(&sse.si_mulMat3x4InPlace), .name = "mulMat3x4InPlace" },
|
||||
.{ .target = 0x6720F0, .replacement = @intFromPtr(&sse.si_normalizeVec3InPlace), .name = "normalizeVec3InPlace" },
|
||||
.{ .target = 0x71BC70, .replacement = @intFromPtr(&sse.si_addVec3ToAccumulator), .name = "addVec3ToAccumulator" },
|
||||
.{ .target = 0x71BF60, .replacement = @intFromPtr(&sse.si_addToColorAccumulator), .name = "addToColorAccumulator" },
|
||||
.{ .target = 0x7B7A80, .replacement = @intFromPtr(&sse.si_packParticleColor), .name = "packParticleColor" },
|
||||
.{ .target = 0x7B7B10, .replacement = @intFromPtr(&sse.si_setParticleAlpha), .name = "setParticleAlpha" },
|
||||
.{ .target = 0x40A2B0, .replacement = @intFromPtr(&sse.si_ftol), .name = "__ftol", .direct_size = 9 },
|
||||
.{ .target = 0x686820, .replacement = @intFromPtr(&sse.si_translateBoundingVol), .name = "translateBoundingVol" },
|
||||
.{ .target = 0x6ABC40, .replacement = @intFromPtr(&sse.si_processLinkedListCollision), .name = "processLinkedListCollision" },
|
||||
.{ .target = 0x686000, .replacement = @intFromPtr(&sse.si_frustumCullBBox), .name = "frustumCullBBox" },
|
||||
.{ .target = 0x6B8C60, .replacement = @intFromPtr(&performSpatialCulling), .name = "PerformSpatialCulling" },
|
||||
.{ .target = 0x6B88E0, .replacement = @intFromPtr(&performCollisionDetectionSSE), .name = "performCollisionDetection" },
|
||||
.{ .target = 0x7C2C40, .replacement = @intFromPtr(&rayTriIntersectIndexedInt), .name = "rayTriIndexedInt" },
|
||||
};
|
||||
|
||||
var count: u32 = 0;
|
||||
for (patches) |p| {
|
||||
if (p.direct_size > 0) {
|
||||
patchDirect(p.target, @ptrFromInt(p.replacement), p.direct_size);
|
||||
} else {
|
||||
patchJmp(p.target, p.replacement);
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Install / Remove
|
||||
// =============================================================================
|
||||
|
||||
pub fn installHooks() void {
|
||||
const result = mod_mutex.acquire(module_name);
|
||||
g_mutex = result.handle;
|
||||
g_is_hook_owner = result.is_owner;
|
||||
if (!g_is_hook_owner) return;
|
||||
|
||||
log = logging.Logger.open(module_name, .both);
|
||||
var installed: u32 = 0;
|
||||
|
||||
// Bone transform SSE
|
||||
if (transform_hook.attach(0x714260, &transformDetour) == .ok) installed += 1;
|
||||
|
||||
// Particle rendering SSE
|
||||
if (particle_hook.attach(0x7B2A50, &particleDetour) == .ok) installed += 1;
|
||||
|
||||
// Glyph cache
|
||||
// Glyph cache removed -- game has internal glyph cache, our hook only sees misses (~30/frame)
|
||||
|
||||
// GUID lookup cache -- A/B testing via transform44
|
||||
// if (findguid_hook.attach(0x464890, &findguidDetour) == .ok) installed += 1;
|
||||
// if (obj_delete_hook.attach(0x464920, &objDeleteDetour) == .ok) installed += 1;
|
||||
|
||||
// GUID cache disabled for now
|
||||
// if (destroy_objmgr_hook.attach(0x467700, &destroyObjMgrDetour) == .ok) installed += 1;
|
||||
|
||||
// Per-frame cache reset
|
||||
if (world_update_hook.attach(0x482EA0, &worldUpdateDetour) == .ok) installed += 1;
|
||||
|
||||
// Silicon SSE binary patches
|
||||
_ = installPatches();
|
||||
|
||||
// MPQ file cache
|
||||
if (filecache.install()) installed += 1;
|
||||
|
||||
// libdeflate inflate replacement
|
||||
if (inflate_hook.install()) installed += 1;
|
||||
|
||||
}
|
||||
|
||||
pub fn lateInit() void {
|
||||
if (!g_is_hook_owner) return;
|
||||
timer_fix.init();
|
||||
}
|
||||
|
||||
pub fn removeHooks() void {
|
||||
if (g_is_hook_owner) {
|
||||
filecache.remove();
|
||||
inflate_hook.remove();
|
||||
transform_hook.detach();
|
||||
particle_hook.detach();
|
||||
// glyph_hook removed
|
||||
findguid_hook.detach();
|
||||
obj_delete_hook.detach();
|
||||
destroy_objmgr_hook.detach();
|
||||
world_update_hook.detach();
|
||||
log.close();
|
||||
mod_mutex.release(&g_mutex);
|
||||
}
|
||||
g_is_hook_owner = false;
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
/* zconf.h -- configuration of the zlib compression library
|
||||
* Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler
|
||||
* For conditions of distribution and use, see copyright notice in zlib.h
|
||||
*/
|
||||
|
||||
#ifndef ZCONF_H
|
||||
#define ZCONF_H
|
||||
|
||||
#include "zlib_name_mangling.h"
|
||||
|
||||
#if !defined(_WIN32) && defined(__WIN32__)
|
||||
# define _WIN32
|
||||
#endif
|
||||
|
||||
/* Clang macro for detecting declspec support
|
||||
* https://clang.llvm.org/docs/LanguageExtensions.html#has-declspec-attribute
|
||||
*/
|
||||
#ifndef __has_declspec_attribute
|
||||
# define __has_declspec_attribute(x) 0
|
||||
#endif
|
||||
|
||||
#if defined(ZLIB_CONST) && !defined(z_const)
|
||||
# define z_const const
|
||||
#else
|
||||
# define z_const
|
||||
#endif
|
||||
|
||||
/* Maximum value for memLevel in deflateInit2 */
|
||||
#ifndef MAX_MEM_LEVEL
|
||||
# define MAX_MEM_LEVEL 9
|
||||
#endif
|
||||
|
||||
/* Maximum value for windowBits in deflateInit2 and inflateInit2.
|
||||
* WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files
|
||||
* created by gzip. (Files created by minigzip can still be extracted by
|
||||
* gzip.)
|
||||
*/
|
||||
#ifndef MIN_WBITS
|
||||
# define MIN_WBITS 8 /* 256 LZ77 window */
|
||||
#endif
|
||||
#ifndef MAX_WBITS
|
||||
# define MAX_WBITS 15 /* 32K LZ77 window */
|
||||
#endif
|
||||
|
||||
/* The memory requirements for deflate are (in bytes):
|
||||
(1 << (windowBits+2)) + (1 << (memLevel+9))
|
||||
that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values)
|
||||
plus a few kilobytes for small objects. For example, if you want to reduce
|
||||
the default memory requirements from 256K to 128K, compile with
|
||||
make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7"
|
||||
Of course this will generally degrade compression (there's no free lunch).
|
||||
|
||||
The memory requirements for inflate are (in bytes) 1 << windowBits
|
||||
that is, 32K for windowBits=15 (default value) plus about 7 kilobytes
|
||||
for small objects.
|
||||
*/
|
||||
|
||||
/* Type declarations */
|
||||
|
||||
|
||||
#ifndef OF /* function prototypes */
|
||||
# define OF(args) args
|
||||
#endif
|
||||
|
||||
#ifdef ZLIB_INTERNAL
|
||||
# define Z_INTERNAL ZLIB_INTERNAL
|
||||
#endif
|
||||
|
||||
/* If building or using zlib as a DLL, define ZLIB_DLL.
|
||||
* This is not mandatory, but it offers a little performance increase.
|
||||
*/
|
||||
#if defined(ZLIB_DLL) && (defined(_WIN32) || (__has_declspec_attribute(dllexport) && __has_declspec_attribute(dllimport)))
|
||||
# ifdef Z_INTERNAL
|
||||
# define Z_EXTERN extern __declspec(dllexport)
|
||||
# else
|
||||
# define Z_EXTERN extern __declspec(dllimport)
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* If building or using zlib with the WINAPI/WINAPIV calling convention,
|
||||
* define ZLIB_WINAPI.
|
||||
* Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI.
|
||||
*/
|
||||
#if defined(ZLIB_WINAPI) && defined(_WIN32)
|
||||
# ifndef WIN32_LEAN_AND_MEAN
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
# endif
|
||||
# include <windows.h>
|
||||
/* No need for _export, use ZLIB.DEF instead. */
|
||||
/* For complete Windows compatibility, use WINAPI, not __stdcall. */
|
||||
# define Z_EXPORT WINAPI
|
||||
# define Z_EXPORTVA WINAPIV
|
||||
#endif
|
||||
|
||||
#ifndef Z_EXTERN
|
||||
# define Z_EXTERN extern
|
||||
#endif
|
||||
#ifndef Z_EXPORT
|
||||
# define Z_EXPORT
|
||||
#endif
|
||||
#ifndef Z_EXPORTVA
|
||||
# define Z_EXPORTVA
|
||||
#endif
|
||||
|
||||
/* Conditional exports */
|
||||
#define ZNG_CONDEXPORT Z_INTERNAL
|
||||
|
||||
/* For backwards compatibility */
|
||||
|
||||
#ifndef ZEXTERN
|
||||
# define ZEXTERN Z_EXTERN
|
||||
#endif
|
||||
#ifndef ZEXPORT
|
||||
# define ZEXPORT Z_EXPORT
|
||||
#endif
|
||||
#ifndef ZEXPORTVA
|
||||
# define ZEXPORTVA Z_EXPORTVA
|
||||
#endif
|
||||
#ifndef FAR
|
||||
# define FAR
|
||||
#endif
|
||||
|
||||
/* Legacy zlib typedefs for backwards compatibility. Don't assume stdint.h is defined. */
|
||||
typedef unsigned char Byte;
|
||||
typedef Byte Bytef;
|
||||
|
||||
typedef unsigned int uInt; /* 16 bits or more */
|
||||
typedef unsigned long uLong; /* 32 bits or more */
|
||||
|
||||
typedef char charf;
|
||||
typedef int intf;
|
||||
typedef uInt uIntf;
|
||||
typedef uLong uLongf;
|
||||
|
||||
typedef void const *voidpc;
|
||||
typedef void *voidpf;
|
||||
typedef void *voidp;
|
||||
|
||||
typedef unsigned int z_crc_t;
|
||||
|
||||
#if 1 /* was set to #if 1 by configure/cmake/etc */
|
||||
# define Z_HAVE_UNISTD_H
|
||||
#endif
|
||||
|
||||
#ifdef NEED_PTRDIFF_T /* may be set to #if 1 by configure/cmake/etc */
|
||||
typedef PTRDIFF_TYPE ptrdiff_t;
|
||||
#endif
|
||||
|
||||
#include <sys/types.h> /* for off_t */
|
||||
|
||||
#include <stddef.h> /* for wchar_t and NULL */
|
||||
|
||||
/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and
|
||||
* "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even
|
||||
* though the former does not conform to the LFS document), but considering
|
||||
* both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as
|
||||
* equivalently requesting no 64-bit operations
|
||||
*/
|
||||
#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1
|
||||
# undef _LARGEFILE64_SOURCE
|
||||
#endif
|
||||
|
||||
#if defined(Z_HAVE_UNISTD_H)
|
||||
# include <unistd.h> /* for SEEK_*, off_t, and _LFS64_LARGEFILE */
|
||||
# ifndef z_off_t
|
||||
# define z_off_t off_t
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0
|
||||
# define Z_LFS64
|
||||
#endif
|
||||
|
||||
#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64)
|
||||
# define Z_LARGE64
|
||||
#endif
|
||||
|
||||
#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64)
|
||||
# define Z_WANT64
|
||||
#endif
|
||||
|
||||
#if !defined(SEEK_SET)
|
||||
# define SEEK_SET 0 /* Seek from beginning of file. */
|
||||
# define SEEK_CUR 1 /* Seek from current position. */
|
||||
# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */
|
||||
#endif
|
||||
|
||||
#ifndef z_off_t
|
||||
# define z_off_t long
|
||||
#endif
|
||||
|
||||
#if !defined(_WIN32) && defined(Z_LARGE64)
|
||||
# define z_off64_t off64_t
|
||||
#else
|
||||
# if defined(__MSYS__)
|
||||
# define z_off64_t _off64_t
|
||||
# elif defined(_WIN32) && !defined(__GNUC__)
|
||||
# define z_off64_t __int64
|
||||
# else
|
||||
# define z_off64_t z_off_t
|
||||
# endif
|
||||
#endif
|
||||
|
||||
typedef size_t z_size_t;
|
||||
|
||||
#endif /* ZCONF_H */
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user