Dusk/WeirdUtils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

perf: vendor libdeflate for fast inflate replacement

Browse Source 
Vendored libdeflate decompress-only C sources into src/performance/.
Compiled as static lib (x86-windows-gnu for libc headers), linked into
the DLL. AVX-512 disabled (not available on 32-bit x86), SSE2/AVX2
paths active.

Target: replace WoW's embedded zlib inflate (~3% CPU) with libdeflate's
~2.3x faster implementation. Hook integration next.
This commit is contained in:
MarcelineVQ
2026-03-24 02:27:51 -07:00
parent 62bcf11d58
commit fb27ff54a0
20 changed files with 5659 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.zig
+33
View File
@@ -151,6 +151,39 @@ pub fn build(b: *std.Build) void {
lib.root_module.addObject(silicon_sse_obj);
lib.root_module.addObject(particle_sse_obj);
lib.root_module.addObject(particle_ref_obj);
// libdeflate — vendored C sources, decompress-only.
// Built as static library targeting x86-windows-gnu (has libc headers).
const libdeflate = b.addLibrary(.{
.linkage = .static,
.name = "deflate",
.root_module = b.createModule(.{
.root_source_file = null,
.target = b.resolveTargetQuery(.{
.cpu_arch = .x86,
.os_tag = .windows,
.abi = .gnu,
}),
.optimize = .ReleaseFast,
.link_libc = true,
}),
});
libdeflate.root_module.addCSourceFiles(.{
.files = &.{
"src/performance/libdeflate/lib/deflate_decompress.c",
"src/performance/libdeflate/lib/zlib_decompress.c",
"src/performance/libdeflate/lib/utils.c",
"src/performance/libdeflate/lib/adler32.c",
"src/performance/libdeflate/lib/x86/cpu_features.c",
},
// Disable AVX-512 codepaths — 512-bit intrinsics require evex512 which
// isn't available on 32-bit x86. AVX/AVX2/SSE paths remain active.
.flags = &.{"-DLIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX512VNNI"},
});
libdeflate.root_module.addIncludePath(b.path("src/performance/libdeflate"));
libdeflate.root_module.addIncludePath(b.path("src/performance/libdeflate/lib"));
lib.root_module.addObjectFile(libdeflate.getEmittedBin());
b.installArtifact(lib);
// Benchmark harness — native x86 Linux executable for profiling SSE replacements
src/performance/libdeflate/common_defs.h
+749
View File
@@ -0,0 +1,749 @@
/*
* common_defs.h
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef COMMON_DEFS_H
#define COMMON_DEFS_H
#include "libdeflate.h"
#include <stdbool.h>
#include <stddef.h> /* for size_t */
#include <stdint.h>
#ifdef _MSC_VER
# include <intrin.h> /* for _BitScan*() and other intrinsics */
# include <stdlib.h> /* for _byteswap_*() */
/* Disable MSVC warnings that are expected. */
/* /W2 */
# pragma warning(disable : 4146) /* unary minus on unsigned type */
/* /W3 */
# pragma warning(disable : 4018) /* signed/unsigned mismatch */
# pragma warning(disable : 4244) /* possible loss of data */
# pragma warning(disable : 4267) /* possible loss of precision */
# pragma warning(disable : 4310) /* cast truncates constant value */
/* /W4 */
# pragma warning(disable : 4100) /* unreferenced formal parameter */
# pragma warning(disable : 4127) /* conditional expression is constant */
# pragma warning(disable : 4189) /* local variable initialized but not referenced */
# pragma warning(disable : 4232) /* nonstandard extension used */
# pragma warning(disable : 4245) /* conversion from 'int' to 'unsigned int' */
# pragma warning(disable : 4295) /* array too small to include terminating null */
#endif
#ifndef FREESTANDING
# include <string.h> /* for memcpy() */
#endif
/* ========================================================================== */
/* Target architecture */
/* ========================================================================== */
/* If possible, define a compiler-independent ARCH_* macro. */
#undef ARCH_X86_64
#undef ARCH_X86_32
#undef ARCH_ARM64
#undef ARCH_ARM32
#undef ARCH_RISCV
#ifdef _MSC_VER
/* Way too many things are broken in ARM64EC to pretend that it is x86_64. */
# if defined(_M_X64) && !defined(_M_ARM64EC)
# define ARCH_X86_64
# elif defined(_M_IX86)
# define ARCH_X86_32
# elif defined(_M_ARM64)
# define ARCH_ARM64
# elif defined(_M_ARM)
# define ARCH_ARM32
# endif
#else
# if defined(__x86_64__)
# define ARCH_X86_64
# elif defined(__i386__)
# define ARCH_X86_32
# elif defined(__aarch64__)
# define ARCH_ARM64
# elif defined(__arm__)
# define ARCH_ARM32
# elif defined(__riscv)
# define ARCH_RISCV
# endif
#endif
/* ========================================================================== */
/* Type definitions */
/* ========================================================================== */
/* Fixed-width integer types */
typedef uint8_t u8;
typedef uint16_t u16;
typedef uint32_t u32;
typedef uint64_t u64;
typedef int8_t s8;
typedef int16_t s16;
typedef int32_t s32;
typedef int64_t s64;
/* ssize_t, if not available in <sys/types.h> */
#ifdef _MSC_VER
# ifdef _WIN64
typedef long long ssize_t;
# else
typedef long ssize_t;
# endif
#endif
/*
* Word type of the target architecture. Use 'size_t' instead of
* 'unsigned long' to account for platforms such as Windows that use 32-bit
* 'unsigned long' on 64-bit architectures.
*/
typedef size_t machine_word_t;
/* Number of bytes in a word */
#define WORDBYTES ((int)sizeof(machine_word_t))
/* Number of bits in a word */
#define WORDBITS (8 * WORDBYTES)
/* ========================================================================== */
/* Optional compiler features */
/* ========================================================================== */
/* Compiler version checks. Only use when absolutely necessary. */
#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
# define GCC_PREREQ(major, minor) \
(__GNUC__ > (major) || \
(__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
# if !GCC_PREREQ(4, 9)
# error "gcc versions older than 4.9 are no longer supported"
# endif
#else
# define GCC_PREREQ(major, minor) 0
#endif
#ifdef __clang__
# ifdef __apple_build_version__
# define CLANG_PREREQ(major, minor, apple_version) \
(__apple_build_version__ >= (apple_version))
# else
# define CLANG_PREREQ(major, minor, apple_version) \
(__clang_major__ > (major) || \
(__clang_major__ == (major) && __clang_minor__ >= (minor)))
# endif
# if !CLANG_PREREQ(3, 9, 8000000)
# error "clang versions older than 3.9 are no longer supported"
# endif
#else
# define CLANG_PREREQ(major, minor, apple_version) 0
#endif
#ifdef _MSC_VER
# define MSVC_PREREQ(version) (_MSC_VER >= (version))
# if !MSVC_PREREQ(1900)
# error "MSVC versions older than Visual Studio 2015 are no longer supported"
# endif
#else
# define MSVC_PREREQ(version) 0
#endif
/*
* __has_attribute(attribute) - check whether the compiler supports the given
* attribute (and also supports doing the check in the first place). Mostly
* useful just for clang, since gcc didn't add this macro until gcc 5.
*/
#ifndef __has_attribute
# define __has_attribute(attribute) 0
#endif
/*
* __has_builtin(builtin) - check whether the compiler supports the given
* builtin (and also supports doing the check in the first place). Mostly
* useful just for clang, since gcc didn't add this macro until gcc 10.
*/
#ifndef __has_builtin
# define __has_builtin(builtin) 0
#endif
/* inline - suggest that a function be inlined */
#ifdef _MSC_VER
# define inline __inline
#endif /* else assume 'inline' is usable as-is */
/* forceinline - force a function to be inlined, if possible */
#if defined(__GNUC__) || __has_attribute(always_inline)
# define forceinline inline __attribute__((always_inline))
#elif defined(_MSC_VER)
# define forceinline __forceinline
#else
# define forceinline inline
#endif
/* MAYBE_UNUSED - mark a function or variable as maybe unused */
#if defined(__GNUC__) || __has_attribute(unused)
# define MAYBE_UNUSED __attribute__((unused))
#else
# define MAYBE_UNUSED
#endif
/* NORETURN - mark a function as never returning, e.g. due to calling abort() */
#if defined(__GNUC__) || __has_attribute(noreturn)
# define NORETURN __attribute__((noreturn))
#else
# define NORETURN
#endif
/*
* restrict - hint that writes only occur through the given pointer.
*
* Don't use MSVC's __restrict, since it has nonstandard behavior.
* Standard restrict is okay, if it is supported.
*/
#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 201112L)
# if defined(__GNUC__) || defined(__clang__)
# define restrict __restrict__
# else
# define restrict
# endif
#endif /* else assume 'restrict' is usable as-is */
/* likely(expr) - hint that an expression is usually true */
#if defined(__GNUC__) || __has_builtin(__builtin_expect)
# define likely(expr) __builtin_expect(!!(expr), 1)
#else
# define likely(expr) (expr)
#endif
/* unlikely(expr) - hint that an expression is usually false */
#if defined(__GNUC__) || __has_builtin(__builtin_expect)
# define unlikely(expr) __builtin_expect(!!(expr), 0)
#else
# define unlikely(expr) (expr)
#endif
/* prefetchr(addr) - prefetch into L1 cache for read */
#undef prefetchr
#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
# define prefetchr(addr) __builtin_prefetch((addr), 0)
#elif defined(_MSC_VER)
# if defined(ARCH_X86_32) || defined(ARCH_X86_64)
# define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
# elif defined(ARCH_ARM64)
# define prefetchr(addr) __prefetch2((addr), 0x00 /* prfop=PLDL1KEEP */)
# elif defined(ARCH_ARM32)
# define prefetchr(addr) __prefetch(addr)
# endif
#endif
#ifndef prefetchr
# define prefetchr(addr)
#endif
/* prefetchw(addr) - prefetch into L1 cache for write */
#undef prefetchw
#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
# define prefetchw(addr) __builtin_prefetch((addr), 1)
#elif defined(_MSC_VER)
# if defined(ARCH_X86_32) || defined(ARCH_X86_64)
# define prefetchw(addr) _m_prefetchw(addr)
# elif defined(ARCH_ARM64)
# define prefetchw(addr) __prefetch2((addr), 0x10 /* prfop=PSTL1KEEP */)
# elif defined(ARCH_ARM32)
# define prefetchw(addr) __prefetchw(addr)
# endif
#endif
#ifndef prefetchw
# define prefetchw(addr)
#endif
/*
* _aligned_attribute(n) - declare that the annotated variable, or variables of
* the annotated type, must be aligned on n-byte boundaries.
*/
#undef _aligned_attribute
#if defined(__GNUC__) || __has_attribute(aligned)
# define _aligned_attribute(n) __attribute__((aligned(n)))
#elif defined(_MSC_VER)
# define _aligned_attribute(n) __declspec(align(n))
#endif
/*
* _target_attribute(attrs) - override the compilation target for a function.
*
* This accepts one or more comma-separated suffixes to the -m prefix jointly
* forming the name of a machine-dependent option. On gcc-like compilers, this
* enables codegen for the given targets, including arbitrary compiler-generated
* code as well as the corresponding intrinsics. On other compilers this macro
* expands to nothing, though MSVC allows intrinsics to be used anywhere anyway.
*/
#if defined(__GNUC__) || __has_attribute(target)
# define _target_attribute(attrs) __attribute__((target(attrs)))
#else
# define _target_attribute(attrs)
#endif
/* ========================================================================== */
/* Miscellaneous macros */
/* ========================================================================== */
#define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
#define MIN(a, b) ((a) <= (b) ? (a) : (b))
#define MAX(a, b) ((a) >= (b) ? (a) : (b))
#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
#define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
#define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
/* ========================================================================== */
/* Endianness handling */
/* ========================================================================== */
/*
* CPU_IS_LITTLE_ENDIAN() - 1 if the CPU is little endian, or 0 if it is big
* endian. When possible this is a compile-time macro that can be used in
* preprocessor conditionals. As a fallback, a generic method is used that
* can't be used in preprocessor conditionals but should still be optimized out.
*/
#if defined(__BYTE_ORDER__) /* gcc v4.6+ and clang */
# define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#elif defined(_MSC_VER)
# define CPU_IS_LITTLE_ENDIAN() true
#else
static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
{
union {
u32 w;
u8 b;
} u;
u.w = 1;
return u.b;
}
#endif
/* bswap16(v) - swap the bytes of a 16-bit integer */
static forceinline u16 bswap16(u16 v)
{
#if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
return __builtin_bswap16(v);
#elif defined(_MSC_VER)
return _byteswap_ushort(v);
#else
return (v << 8) | (v >> 8);
#endif
}
/* bswap32(v) - swap the bytes of a 32-bit integer */
static forceinline u32 bswap32(u32 v)
{
#if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
return __builtin_bswap32(v);
#elif defined(_MSC_VER)
return _byteswap_ulong(v);
#else
return ((v & 0x000000FF) << 24) |
((v & 0x0000FF00) << 8) |
((v & 0x00FF0000) >> 8) |
((v & 0xFF000000) >> 24);
#endif
}
/* bswap64(v) - swap the bytes of a 64-bit integer */
static forceinline u64 bswap64(u64 v)
{
#if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
return __builtin_bswap64(v);
#elif defined(_MSC_VER)
return _byteswap_uint64(v);
#else
return ((v & 0x00000000000000FF) << 56) |
((v & 0x000000000000FF00) << 40) |
((v & 0x0000000000FF0000) << 24) |
((v & 0x00000000FF000000) << 8) |
((v & 0x000000FF00000000) >> 8) |
((v & 0x0000FF0000000000) >> 24) |
((v & 0x00FF000000000000) >> 40) |
((v & 0xFF00000000000000) >> 56);
#endif
}
#define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
#define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
#define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
#define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
#define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
#define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
/* ========================================================================== */
/* Unaligned memory accesses */
/* ========================================================================== */
/*
* UNALIGNED_ACCESS_IS_FAST() - 1 if unaligned memory accesses can be performed
* efficiently on the target platform, otherwise 0.
*/
#if (defined(__GNUC__) || defined(__clang__)) && \
(defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
defined(__ARM_FEATURE_UNALIGNED) || \
defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
defined(__riscv_misaligned_fast) || \
/*
* For all compilation purposes, WebAssembly behaves like any other CPU
* instruction set. Even though WebAssembly engine might be running on
* top of different actual CPU architectures, the WebAssembly spec
* itself permits unaligned access and it will be fast on most of those
* platforms, and simulated at the engine level on others, so it's
* worth treating it as a CPU architecture with fast unaligned access.
*/ defined(__wasm__))
# define UNALIGNED_ACCESS_IS_FAST 1
#elif defined(_MSC_VER)
# define UNALIGNED_ACCESS_IS_FAST 1
#else
# define UNALIGNED_ACCESS_IS_FAST 0
#endif
/*
* Implementing unaligned memory accesses using memcpy() is portable, and it
* usually gets optimized appropriately by modern compilers. I.e., each
* memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled to a load or store
* instruction, not to an actual function call.
*
* We no longer use the "packed struct" approach to unaligned accesses, as that
* is nonstandard, has unclear semantics, and doesn't receive enough testing
* (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
*
* arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception
* where memcpy() generates inefficient code
* (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366). However, we no longer
* consider that one case important enough to maintain different code for.
* If you run into it, please just use a newer version of gcc (or use clang).
*/
#ifdef FREESTANDING
# define MEMCOPY __builtin_memcpy
#else
# define MEMCOPY memcpy
#endif
/* Unaligned loads and stores without endianness conversion */
#define DEFINE_UNALIGNED_TYPE(type) \
static forceinline type \
load_##type##_unaligned(const void *p) \
{ \
type v; \
\
MEMCOPY(&v, p, sizeof(v)); \
return v; \
} \
\
static forceinline void \
store_##type##_unaligned(type v, void *p) \
{ \
MEMCOPY(p, &v, sizeof(v)); \
}
DEFINE_UNALIGNED_TYPE(u16)
DEFINE_UNALIGNED_TYPE(u32)
DEFINE_UNALIGNED_TYPE(u64)
DEFINE_UNALIGNED_TYPE(machine_word_t)
#undef MEMCOPY
#define load_word_unaligned load_machine_word_t_unaligned
#define store_word_unaligned store_machine_word_t_unaligned
/* Unaligned loads with endianness conversion */
static forceinline u16
get_unaligned_le16(const u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST)
return le16_bswap(load_u16_unaligned(p));
else
return ((u16)p[1] << 8) | p[0];
}
static forceinline u16
get_unaligned_be16(const u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST)
return be16_bswap(load_u16_unaligned(p));
else
return ((u16)p[0] << 8) | p[1];
}
static forceinline u32
get_unaligned_le32(const u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST)
return le32_bswap(load_u32_unaligned(p));
else
return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
((u32)p[1] << 8) | p[0];
}
static forceinline u32
get_unaligned_be32(const u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST)
return be32_bswap(load_u32_unaligned(p));
else
return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
((u32)p[2] << 8) | p[3];
}
static forceinline u64
get_unaligned_le64(const u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST)
return le64_bswap(load_u64_unaligned(p));
else
return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
((u64)p[5] << 40) | ((u64)p[4] << 32) |
((u64)p[3] << 24) | ((u64)p[2] << 16) |
((u64)p[1] << 8) | p[0];
}
static forceinline machine_word_t
get_unaligned_leword(const u8 *p)
{
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
if (WORDBITS == 32)
return get_unaligned_le32(p);
else
return get_unaligned_le64(p);
}
/* Unaligned stores with endianness conversion */
static forceinline void
put_unaligned_le16(u16 v, u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST) {
store_u16_unaligned(le16_bswap(v), p);
} else {
p[0] = (u8)(v >> 0);
p[1] = (u8)(v >> 8);
}
}
static forceinline void
put_unaligned_be16(u16 v, u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST) {
store_u16_unaligned(be16_bswap(v), p);
} else {
p[0] = (u8)(v >> 8);
p[1] = (u8)(v >> 0);
}
}
static forceinline void
put_unaligned_le32(u32 v, u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST) {
store_u32_unaligned(le32_bswap(v), p);
} else {
p[0] = (u8)(v >> 0);
p[1] = (u8)(v >> 8);
p[2] = (u8)(v >> 16);
p[3] = (u8)(v >> 24);
}
}
static forceinline void
put_unaligned_be32(u32 v, u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST) {
store_u32_unaligned(be32_bswap(v), p);
} else {
p[0] = (u8)(v >> 24);
p[1] = (u8)(v >> 16);
p[2] = (u8)(v >> 8);
p[3] = (u8)(v >> 0);
}
}
static forceinline void
put_unaligned_le64(u64 v, u8 *p)
{
if (UNALIGNED_ACCESS_IS_FAST) {
store_u64_unaligned(le64_bswap(v), p);
} else {
p[0] = (u8)(v >> 0);
p[1] = (u8)(v >> 8);
p[2] = (u8)(v >> 16);
p[3] = (u8)(v >> 24);
p[4] = (u8)(v >> 32);
p[5] = (u8)(v >> 40);
p[6] = (u8)(v >> 48);
p[7] = (u8)(v >> 56);
}
}
static forceinline void
put_unaligned_leword(machine_word_t v, u8 *p)
{
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
if (WORDBITS == 32)
put_unaligned_le32(v, p);
else
put_unaligned_le64(v, p);
}
/* ========================================================================== */
/* Bit manipulation functions */
/* ========================================================================== */
/*
* Bit Scan Reverse (BSR) - find the 0-based index (relative to the least
* significant end) of the *most* significant 1 bit in the input value. The
* input value must be nonzero!
*/
static forceinline unsigned
bsr32(u32 v)
{
#if defined(__GNUC__) || __has_builtin(__builtin_clz)
return 31 - __builtin_clz(v);
#elif defined(_MSC_VER)
unsigned long i;
_BitScanReverse(&i, v);
return i;
#else
unsigned i = 0;
while ((v >>= 1) != 0)
i++;
return i;
#endif
}
static forceinline unsigned
bsr64(u64 v)
{
#if defined(__GNUC__) || __has_builtin(__builtin_clzll)
return 63 - __builtin_clzll(v);
#elif defined(_MSC_VER) && defined(_WIN64)
unsigned long i;
_BitScanReverse64(&i, v);
return i;
#else
unsigned i = 0;
while ((v >>= 1) != 0)
i++;
return i;
#endif
}
static forceinline unsigned
bsrw(machine_word_t v)
{
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
if (WORDBITS == 32)
return bsr32(v);
else
return bsr64(v);
}
/*
* Bit Scan Forward (BSF) - find the 0-based index (relative to the least
* significant end) of the *least* significant 1 bit in the input value. The
* input value must be nonzero!
*/
static forceinline unsigned
bsf32(u32 v)
{
#if defined(__GNUC__) || __has_builtin(__builtin_ctz)
return __builtin_ctz(v);
#elif defined(_MSC_VER)
unsigned long i;
_BitScanForward(&i, v);
return i;
#else
unsigned i = 0;
for (; (v & 1) == 0; v >>= 1)
i++;
return i;
#endif
}
static forceinline unsigned
bsf64(u64 v)
{
#if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
return __builtin_ctzll(v);
#elif defined(_MSC_VER) && defined(_WIN64)
unsigned long i;
_BitScanForward64(&i, v);
return i;
#else
unsigned i = 0;
for (; (v & 1) == 0; v >>= 1)
i++;
return i;
#endif
}
static forceinline unsigned
bsfw(machine_word_t v)
{
STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
if (WORDBITS == 32)
return bsf32(v);
else
return bsf64(v);
}
/*
* rbit32(v): reverse the bits in a 32-bit integer. This doesn't have a
* fallback implementation; use '#ifdef rbit32' to check if this is available.
*/
#undef rbit32
#if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
(__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
static forceinline u32
rbit32(u32 v)
{
__asm__("rbit %0, %1" : "=r" (v) : "r" (v));
return v;
}
#define rbit32 rbit32
#elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
static forceinline u32
rbit32(u32 v)
{
__asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
return v;
}
#define rbit32 rbit32
#endif
#endif /* COMMON_DEFS_H */
src/performance/libdeflate/lib/adler32.c
+162
View File
@@ -0,0 +1,162 @@
/*
* adler32.c - Adler-32 checksum algorithm
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "lib_common.h"
/* The Adler-32 divisor, or "base", value */
#define DIVISOR 65521
/*
* MAX_CHUNK_LEN is the most bytes that can be processed without the possibility
* of s2 overflowing when it is represented as an unsigned 32-bit integer. This
* value was computed using the following Python script:
*
* divisor = 65521
* count = 0
* s1 = divisor - 1
* s2 = divisor - 1
* while True:
* s1 += 0xFF
* s2 += s1
* if s2 > 0xFFFFFFFF:
* break
* count += 1
* print(count)
*
* Note that to get the correct worst-case value, we must assume that every byte
* has value 0xFF and that s1 and s2 started with the highest possible values
* modulo the divisor.
*/
#define MAX_CHUNK_LEN 5552
/*
* Update the Adler-32 values s1 and s2 using n bytes from p, update p to p + n,
* update n to 0, and reduce s1 and s2 mod DIVISOR. It is assumed that neither
* s1 nor s2 can overflow before the reduction at the end, i.e. n plus any bytes
* already processed after the last reduction must not exceed MAX_CHUNK_LEN.
*
* This uses only portable C code. This is used as a fallback when a vectorized
* implementation of Adler-32 (e.g. AVX2) is unavailable on the platform.
*
* Some of the vectorized implementations also use this to handle the end of the
* data when the data isn't evenly divisible by the length the vectorized code
* works on. To avoid compiler errors about target-specific option mismatches
* when this is used in that way, this is a macro rather than a function.
*
* Although this is unvectorized, this does include an optimization where the
* main loop processes four bytes at a time using a strategy similar to that
* used by vectorized implementations. This provides increased instruction-
* level parallelism compared to the traditional 's1 += *p++; s2 += s1;'.
*/
#define ADLER32_CHUNK(s1, s2, p, n) \
do { \
if (n >= 4) { \
u32 s1_sum = 0; \
u32 byte_0_sum = 0; \
u32 byte_1_sum = 0; \
u32 byte_2_sum = 0; \
u32 byte_3_sum = 0; \
\
do { \
s1_sum += s1; \
s1 += p[0] + p[1] + p[2] + p[3]; \
byte_0_sum += p[0]; \
byte_1_sum += p[1]; \
byte_2_sum += p[2]; \
byte_3_sum += p[3]; \
p += 4; \
n -= 4; \
} while (n >= 4); \
s2 += (4 * (s1_sum + byte_0_sum)) + (3 * byte_1_sum) + \
(2 * byte_2_sum) + byte_3_sum; \
} \
for (; n; n--, p++) { \
s1 += *p; \
s2 += s1; \
} \
s1 %= DIVISOR; \
s2 %= DIVISOR; \
} while (0)
static u32 MAYBE_UNUSED
adler32_generic(u32 adler, const u8 *p, size_t len)
{
u32 s1 = adler & 0xFFFF;
u32 s2 = adler >> 16;
while (len) {
size_t n = MIN(len, MAX_CHUNK_LEN & ~3);
len -= n;
ADLER32_CHUNK(s1, s2, p, n);
}
return (s2 << 16) | s1;
}
/* Include architecture-specific implementation(s) if available. */
#undef DEFAULT_IMPL
#undef arch_select_adler32_func
typedef u32 (*adler32_func_t)(u32 adler, const u8 *p, size_t len);
#if defined(ARCH_ARM32) || defined(ARCH_ARM64)
# include "arm/adler32_impl.h"
#elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
# include "x86/adler32_impl.h"
#endif
#ifndef DEFAULT_IMPL
# define DEFAULT_IMPL adler32_generic
#endif
#ifdef arch_select_adler32_func
static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len);
static volatile adler32_func_t adler32_impl = dispatch_adler32;
/* Choose the best implementation at runtime. */
static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len)
{
adler32_func_t f = arch_select_adler32_func();
if (f == NULL)
f = DEFAULT_IMPL;
adler32_impl = f;
return f(adler, p, len);
}
#else
/* The best implementation is statically known, so call it directly. */
#define adler32_impl DEFAULT_IMPL
#endif
LIBDEFLATEAPI u32
libdeflate_adler32(u32 adler, const void *buffer, size_t len)
{
if (buffer == NULL) /* Return initial value. */
return 1;
return adler32_impl(adler, buffer, len);
}
src/performance/libdeflate/lib/cpu_features_common.h
+93
View File
@@ -0,0 +1,93 @@
/*
* cpu_features_common.h - code shared by all lib/$arch/cpu_features.c
*
* Copyright 2020 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef LIB_CPU_FEATURES_COMMON_H
#define LIB_CPU_FEATURES_COMMON_H
#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
/* for strdup() and strtok_r() */
# undef _ANSI_SOURCE
# ifndef __APPLE__
# undef _GNU_SOURCE
# define _GNU_SOURCE
# endif
# include <stdio.h>
# include <stdlib.h>
# include <string.h>
#endif
#include "lib_common.h"
struct cpu_feature {
u32 bit;
const char *name;
};
#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
/* Disable any features that are listed in $LIBDEFLATE_DISABLE_CPU_FEATURES. */
static inline void
disable_cpu_features_for_testing(u32 *features,
const struct cpu_feature *feature_table,
size_t feature_table_length)
{
char *env_value, *strbuf, *p, *saveptr = NULL;
size_t i;
env_value = getenv("LIBDEFLATE_DISABLE_CPU_FEATURES");
if (!env_value)
return;
strbuf = strdup(env_value);
if (!strbuf)
abort();
p = strtok_r(strbuf, ",", &saveptr);
while (p) {
for (i = 0; i < feature_table_length; i++) {
if (strcmp(p, feature_table[i].name) == 0) {
*features &= ~feature_table[i].bit;
break;
}
}
if (i == feature_table_length) {
fprintf(stderr,
"unrecognized feature in LIBDEFLATE_DISABLE_CPU_FEATURES: \"%s\"\n",
p);
abort();
}
p = strtok_r(NULL, ",", &saveptr);
}
free(strbuf);
}
#else /* TEST_SUPPORT__DO_NOT_USE */
static inline void
disable_cpu_features_for_testing(u32 *features,
const struct cpu_feature *feature_table,
size_t feature_table_length)
{
}
#endif /* !TEST_SUPPORT__DO_NOT_USE */
#endif /* LIB_CPU_FEATURES_COMMON_H */
src/performance/libdeflate/lib/decompress_template.h
+777
View File
@@ -0,0 +1,777 @@
/*
* decompress_template.h
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
/*
* This is the actual DEFLATE decompression routine, lifted out of
* deflate_decompress.c so that it can be compiled multiple times with different
* target instruction sets.
*/
#ifndef ATTRIBUTES
# define ATTRIBUTES
#endif
#ifndef EXTRACT_VARBITS
# define EXTRACT_VARBITS(word, count) ((word) & BITMASK(count))
#endif
#ifndef EXTRACT_VARBITS8
# define EXTRACT_VARBITS8(word, count) ((word) & BITMASK((u8)(count)))
#endif
static ATTRIBUTES MAYBE_UNUSED enum libdeflate_result
FUNCNAME(struct libdeflate_decompressor * restrict d,
const void * restrict in, size_t in_nbytes,
void * restrict out, size_t out_nbytes_avail,
size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
{
u8 *out_next = out;
u8 * const out_end = out_next + out_nbytes_avail;
u8 * const out_fastloop_end =
out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN);
/* Input bitstream state; see deflate_decompress.c for documentation */
const u8 *in_next = in;
const u8 * const in_end = in_next + in_nbytes;
const u8 * const in_fastloop_end =
in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ);
bitbuf_t bitbuf = 0;
bitbuf_t saved_bitbuf;
u32 bitsleft = 0;
size_t overread_count = 0;
bool is_final_block;
unsigned block_type;
unsigned num_litlen_syms;
unsigned num_offset_syms;
bitbuf_t litlen_tablemask;
u32 entry;
next_block:
/* Starting to read the next block */
;
STATIC_ASSERT(CAN_CONSUME(1 + 2 + 5 + 5 + 4 + 3));
REFILL_BITS();
/* BFINAL: 1 bit */
is_final_block = bitbuf & BITMASK(1);
/* BTYPE: 2 bits */
block_type = (bitbuf >> 1) & BITMASK(2);
if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) {
/* Dynamic Huffman block */
/* The order in which precode lengths are stored */
static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = {
16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
};
unsigned num_explicit_precode_lens;
unsigned i;
/* Read the codeword length counts. */
STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 257 + BITMASK(5));
num_litlen_syms = 257 + ((bitbuf >> 3) & BITMASK(5));
STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 1 + BITMASK(5));
num_offset_syms = 1 + ((bitbuf >> 8) & BITMASK(5));
STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == 4 + BITMASK(4));
num_explicit_precode_lens = 4 + ((bitbuf >> 13) & BITMASK(4));
d->static_codes_loaded = false;
/*
* Read the precode codeword lengths.
*
* A 64-bit bitbuffer is just one bit too small to hold the
* maximum number of precode lens, so to minimize branches we
* merge one len with the previous fields.
*/
STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);
if (CAN_CONSUME(3 * (DEFLATE_NUM_PRECODE_SYMS - 1))) {
d->u.precode_lens[deflate_precode_lens_permutation[0]] =
(bitbuf >> 17) & BITMASK(3);
bitbuf >>= 20;
bitsleft -= 20;
REFILL_BITS();
i = 1;
do {
d->u.precode_lens[deflate_precode_lens_permutation[i]] =
bitbuf & BITMASK(3);
bitbuf >>= 3;
bitsleft -= 3;
} while (++i < num_explicit_precode_lens);
} else {
bitbuf >>= 17;
bitsleft -= 17;
i = 0;
do {
if ((u8)bitsleft < 3)
REFILL_BITS();
d->u.precode_lens[deflate_precode_lens_permutation[i]] =
bitbuf & BITMASK(3);
bitbuf >>= 3;
bitsleft -= 3;
} while (++i < num_explicit_precode_lens);
}
for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)
d->u.precode_lens[deflate_precode_lens_permutation[i]] = 0;
/* Build the decode table for the precode. */
SAFETY_CHECK(build_precode_decode_table(d));
/* Decode the litlen and offset codeword lengths. */
i = 0;
do {
unsigned presym;
u8 rep_val;
unsigned rep_count;
if ((u8)bitsleft < DEFLATE_MAX_PRE_CODEWORD_LEN + 7)
REFILL_BITS();
/*
* The code below assumes that the precode decode table
* doesn't have any subtables.
*/
STATIC_ASSERT(PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN);
/* Decode the next precode symbol. */
entry = d->u.l.precode_decode_table[
bitbuf & BITMASK(DEFLATE_MAX_PRE_CODEWORD_LEN)];
bitbuf >>= (u8)entry;
bitsleft -= entry; /* optimization: subtract full entry */
presym = entry >> 16;
if (presym < 16) {
/* Explicit codeword length */
d->u.l.lens[i++] = presym;
continue;
}
/* Run-length encoded codeword lengths */
/*
* Note: we don't need to immediately verify that the
* repeat count doesn't overflow the number of elements,
* since we've sized the lens array to have enough extra
* space to allow for the worst-case overrun (138 zeroes
* when only 1 length was remaining).
*
* In the case of the small repeat counts (presyms 16
* and 17), it is fastest to always write the maximum
* number of entries. That gets rid of branches that
* would otherwise be required.
*
* It is not just because of the numerical order that
* our checks go in the order 'presym < 16', 'presym ==
* 16', and 'presym == 17'. For typical data this is
* ordered from most frequent to least frequent case.
*/
STATIC_ASSERT(DEFLATE_MAX_LENS_OVERRUN == 138 - 1);
if (presym == 16) {
/* Repeat the previous length 3 - 6 times. */
SAFETY_CHECK(i != 0);
rep_val = d->u.l.lens[i - 1];
STATIC_ASSERT(3 + BITMASK(2) == 6);
rep_count = 3 + (bitbuf & BITMASK(2));
bitbuf >>= 2;
bitsleft -= 2;
d->u.l.lens[i + 0] = rep_val;
d->u.l.lens[i + 1] = rep_val;
d->u.l.lens[i + 2] = rep_val;
d->u.l.lens[i + 3] = rep_val;
d->u.l.lens[i + 4] = rep_val;
d->u.l.lens[i + 5] = rep_val;
i += rep_count;
} else if (presym == 17) {
/* Repeat zero 3 - 10 times. */
STATIC_ASSERT(3 + BITMASK(3) == 10);
rep_count = 3 + (bitbuf & BITMASK(3));
bitbuf >>= 3;
bitsleft -= 3;
d->u.l.lens[i + 0] = 0;
d->u.l.lens[i + 1] = 0;
d->u.l.lens[i + 2] = 0;
d->u.l.lens[i + 3] = 0;
d->u.l.lens[i + 4] = 0;
d->u.l.lens[i + 5] = 0;
d->u.l.lens[i + 6] = 0;
d->u.l.lens[i + 7] = 0;
d->u.l.lens[i + 8] = 0;
d->u.l.lens[i + 9] = 0;
i += rep_count;
} else {
/* Repeat zero 11 - 138 times. */
STATIC_ASSERT(11 + BITMASK(7) == 138);
rep_count = 11 + (bitbuf & BITMASK(7));
bitbuf >>= 7;
bitsleft -= 7;
memset(&d->u.l.lens[i], 0,
rep_count * sizeof(d->u.l.lens[i]));
i += rep_count;
}
} while (i < num_litlen_syms + num_offset_syms);
/* Unnecessary, but check this for consistency with zlib. */
SAFETY_CHECK(i == num_litlen_syms + num_offset_syms);
} else if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) {
u16 len, nlen;
/*
* Uncompressed block: copy 'len' bytes literally from the input
* buffer to the output buffer.
*/
bitsleft -= 3; /* for BTYPE and BFINAL */
/*
* Align the bitstream to the next byte boundary. This means
* the next byte boundary as if we were reading a byte at a
* time. Therefore, we have to rewind 'in_next' by any bytes
* that have been refilled but not actually consumed yet (not
* counting overread bytes, which don't increment 'in_next').
*/
bitsleft = (u8)bitsleft;
SAFETY_CHECK(overread_count <= (bitsleft >> 3));
in_next -= (bitsleft >> 3) - overread_count;
overread_count = 0;
bitbuf = 0;
bitsleft = 0;
SAFETY_CHECK(in_end - in_next >= 4);
len = get_unaligned_le16(in_next);
nlen = get_unaligned_le16(in_next + 2);
in_next += 4;
SAFETY_CHECK(len == (u16)~nlen);
if (unlikely(len > out_end - out_next))
return LIBDEFLATE_INSUFFICIENT_SPACE;
SAFETY_CHECK(len <= in_end - in_next);
memcpy(out_next, in_next, len);
in_next += len;
out_next += len;
goto block_done;
} else {
unsigned i;
SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);
/*
* Static Huffman block: build the decode tables for the static
* codes. Skip doing so if the tables are already set up from
* an earlier static block; this speeds up decompression of
* degenerate input of many empty or very short static blocks.
*
* Afterwards, the remainder is the same as decompressing a
* dynamic Huffman block.
*/
bitbuf >>= 3; /* for BTYPE and BFINAL */
bitsleft -= 3;
if (d->static_codes_loaded)
goto have_decode_tables;
d->static_codes_loaded = true;
STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 288);
STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 32);
for (i = 0; i < 144; i++)
d->u.l.lens[i] = 8;
for (; i < 256; i++)
d->u.l.lens[i] = 9;
for (; i < 280; i++)
d->u.l.lens[i] = 7;
for (; i < 288; i++)
d->u.l.lens[i] = 8;
for (; i < 288 + 32; i++)
d->u.l.lens[i] = 5;
num_litlen_syms = 288;
num_offset_syms = 32;
}
/* Decompressing a Huffman block (either dynamic or static) */
SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));
SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));
have_decode_tables:
litlen_tablemask = BITMASK(d->litlen_tablebits);
/*
* This is the "fastloop" for decoding literals and matches. It does
* bounds checks on in_next and out_next in the loop conditions so that
* additional bounds checks aren't needed inside the loop body.
*
* To reduce latency, the bitbuffer is refilled and the next litlen
* decode table entry is preloaded before each loop iteration.
*/
if (in_next >= in_fastloop_end || out_next >= out_fastloop_end)
goto generic_loop;
REFILL_BITS_IN_FASTLOOP();
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
do {
u32 length, offset, lit;
const u8 *src;
u8 *dst;
/*
* Consume the bits for the litlen decode table entry. Save the
* original bitbuf for later, in case the extra match length
* bits need to be extracted from it.
*/
saved_bitbuf = bitbuf;
bitbuf >>= (u8)entry;
bitsleft -= entry; /* optimization: subtract full entry */
/*
* Begin by checking for a "fast" literal, i.e. a literal that
* doesn't need a subtable.
*/
if (entry & HUFFDEC_LITERAL) {
/*
* On 64-bit platforms, we decode up to 2 extra fast
* literals in addition to the primary item, as this
* increases performance and still leaves enough bits
* remaining for what follows. We could actually do 3,
* assuming LITLEN_TABLEBITS=11, but that actually
* decreases performance slightly (perhaps by messing
* with the branch prediction of the conditional refill
* that happens later while decoding the match offset).
*
* Note: the definitions of FASTLOOP_MAX_BYTES_WRITTEN
* and FASTLOOP_MAX_BYTES_READ need to be updated if the
* number of extra literals decoded here is changed.
*/
if (/* enough bits for 2 fast literals + length + offset preload? */
CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
LENGTH_MAXBITS,
OFFSET_TABLEBITS) &&
/* enough bits for 2 fast literals + slow literal + litlen preload? */
CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
DEFLATE_MAX_LITLEN_CODEWORD_LEN,
LITLEN_TABLEBITS)) {
/* 1st extra fast literal */
lit = entry >> 16;
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
saved_bitbuf = bitbuf;
bitbuf >>= (u8)entry;
bitsleft -= entry;
*out_next++ = lit;
if (entry & HUFFDEC_LITERAL) {
/* 2nd extra fast literal */
lit = entry >> 16;
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
saved_bitbuf = bitbuf;
bitbuf >>= (u8)entry;
bitsleft -= entry;
*out_next++ = lit;
if (entry & HUFFDEC_LITERAL) {
/*
* Another fast literal, but
* this one is in lieu of the
* primary item, so it doesn't
* count as one of the extras.
*/
lit = entry >> 16;
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
REFILL_BITS_IN_FASTLOOP();
*out_next++ = lit;
continue;
}
}
} else {
/*
* Decode a literal. While doing so, preload
* the next litlen decode table entry and refill
* the bitbuffer. To reduce latency, we've
* arranged for there to be enough "preloadable"
* bits remaining to do the table preload
* independently of the refill.
*/
STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(
LITLEN_TABLEBITS, LITLEN_TABLEBITS));
lit = entry >> 16;
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
REFILL_BITS_IN_FASTLOOP();
*out_next++ = lit;
continue;
}
}
/*
* It's not a literal entry, so it can be a length entry, a
* subtable pointer entry, or an end-of-block entry. Detect the
* two unlikely cases by testing the HUFFDEC_EXCEPTIONAL flag.
*/
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
/* Subtable pointer or end-of-block entry */
if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
goto block_done;
/*
* A subtable is required. Load and consume the
* subtable entry. The subtable entry can be of any
* type: literal, length, or end-of-block.
*/
entry = d->u.litlen_decode_table[(entry >> 16) +
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
saved_bitbuf = bitbuf;
bitbuf >>= (u8)entry;
bitsleft -= entry;
/*
* 32-bit platforms that use the byte-at-a-time refill
* method have to do a refill here for there to always
* be enough bits to decode a literal that requires a
* subtable, then preload the next litlen decode table
* entry; or to decode a match length that requires a
* subtable, then preload the offset decode table entry.
*/
if (!CAN_CONSUME_AND_THEN_PRELOAD(DEFLATE_MAX_LITLEN_CODEWORD_LEN,
LITLEN_TABLEBITS) ||
!CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXBITS,
OFFSET_TABLEBITS))
REFILL_BITS_IN_FASTLOOP();
if (entry & HUFFDEC_LITERAL) {
/* Decode a literal that required a subtable. */
lit = entry >> 16;
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
REFILL_BITS_IN_FASTLOOP();
*out_next++ = lit;
continue;
}
if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
goto block_done;
/* Else, it's a length that required a subtable. */
}
/*
* Decode the match length: the length base value associated
* with the litlen symbol (which we extract from the decode
* table entry), plus the extra length bits. We don't need to
* consume the extra length bits here, as they were included in
* the bits consumed by the entry earlier. We also don't need
* to check for too-long matches here, as this is inside the
* fastloop where it's already been verified that the output
* buffer has enough space remaining to copy a max-length match.
*/
length = entry >> 16;
length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
/*
* Decode the match offset. There are enough "preloadable" bits
* remaining to preload the offset decode table entry, but a
* refill might be needed before consuming it.
*/
STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXFASTBITS,
OFFSET_TABLEBITS));
entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
if (CAN_CONSUME_AND_THEN_PRELOAD(OFFSET_MAXBITS,
LITLEN_TABLEBITS)) {
/*
* Decoding a match offset on a 64-bit platform. We may
* need to refill once, but then we can decode the whole
* offset and preload the next litlen table entry.
*/
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
/* Offset codeword requires a subtable */
if (unlikely((u8)bitsleft < OFFSET_MAXBITS +
LITLEN_TABLEBITS - PRELOAD_SLACK))
REFILL_BITS_IN_FASTLOOP();
bitbuf >>= OFFSET_TABLEBITS;
bitsleft -= OFFSET_TABLEBITS;
entry = d->offset_decode_table[(entry >> 16) +
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
} else if (unlikely((u8)bitsleft < OFFSET_MAXFASTBITS +
LITLEN_TABLEBITS - PRELOAD_SLACK))
REFILL_BITS_IN_FASTLOOP();
} else {
/* Decoding a match offset on a 32-bit platform */
REFILL_BITS_IN_FASTLOOP();
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
/* Offset codeword requires a subtable */
bitbuf >>= OFFSET_TABLEBITS;
bitsleft -= OFFSET_TABLEBITS;
entry = d->offset_decode_table[(entry >> 16) +
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
REFILL_BITS_IN_FASTLOOP();
/* No further refill needed before extra bits */
STATIC_ASSERT(CAN_CONSUME(
OFFSET_MAXBITS - OFFSET_TABLEBITS));
} else {
/* No refill needed before extra bits */
STATIC_ASSERT(CAN_CONSUME(OFFSET_MAXFASTBITS));
}
}
saved_bitbuf = bitbuf;
bitbuf >>= (u8)entry;
bitsleft -= entry; /* optimization: subtract full entry */
offset = entry >> 16;
offset += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
/* Validate the match offset; needed even in the fastloop. */
SAFETY_CHECK(offset <= out_next - (const u8 *)out);
src = out_next - offset;
dst = out_next;
out_next += length;
/*
* Before starting to issue the instructions to copy the match,
* refill the bitbuffer and preload the litlen decode table
* entry for the next loop iteration. This can increase
* performance by allowing the latency of the match copy to
* overlap with these other operations. To further reduce
* latency, we've arranged for there to be enough bits remaining
* to do the table preload independently of the refill, except
* on 32-bit platforms using the byte-at-a-time refill method.
*/
if (!CAN_CONSUME_AND_THEN_PRELOAD(
MAX(OFFSET_MAXBITS - OFFSET_TABLEBITS,
OFFSET_MAXFASTBITS),
LITLEN_TABLEBITS) &&
unlikely((u8)bitsleft < LITLEN_TABLEBITS - PRELOAD_SLACK))
REFILL_BITS_IN_FASTLOOP();
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
REFILL_BITS_IN_FASTLOOP();
/*
* Copy the match. On most CPUs the fastest method is a
* word-at-a-time copy, unconditionally copying about 5 words
* since this is enough for most matches without being too much.
*
* The normal word-at-a-time copy works for offset >= WORDBYTES,
* which is most cases. The case of offset == 1 is also common
* and is worth optimizing for, since it is just RLE encoding of
* the previous byte, which is the result of compressing long
* runs of the same byte.
*
* Writing past the match 'length' is allowed here, since it's
* been ensured there is enough output space left for a slight
* overrun. FASTLOOP_MAX_BYTES_WRITTEN needs to be updated if
* the maximum possible overrun here is changed.
*/
if (UNALIGNED_ACCESS_IS_FAST && offset >= WORDBYTES) {
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
while (dst < out_next) {
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
store_word_unaligned(load_word_unaligned(src), dst);
src += WORDBYTES;
dst += WORDBYTES;
}
} else if (UNALIGNED_ACCESS_IS_FAST && offset == 1) {
machine_word_t v;
/*
* This part tends to get auto-vectorized, so keep it
* copying a multiple of 16 bytes at a time.
*/
v = (machine_word_t)0x0101010101010101 * src[0];
store_word_unaligned(v, dst);
dst += WORDBYTES;
store_word_unaligned(v, dst);
dst += WORDBYTES;
store_word_unaligned(v, dst);
dst += WORDBYTES;
store_word_unaligned(v, dst);
dst += WORDBYTES;
while (dst < out_next) {
store_word_unaligned(v, dst);
dst += WORDBYTES;
store_word_unaligned(v, dst);
dst += WORDBYTES;
store_word_unaligned(v, dst);
dst += WORDBYTES;
store_word_unaligned(v, dst);
dst += WORDBYTES;
}
} else if (UNALIGNED_ACCESS_IS_FAST) {
store_word_unaligned(load_word_unaligned(src), dst);
src += offset;
dst += offset;
store_word_unaligned(load_word_unaligned(src), dst);
src += offset;
dst += offset;
do {
store_word_unaligned(load_word_unaligned(src), dst);
src += offset;
dst += offset;
store_word_unaligned(load_word_unaligned(src), dst);
src += offset;
dst += offset;
} while (dst < out_next);
} else {
*dst++ = *src++;
*dst++ = *src++;
do {
*dst++ = *src++;
} while (dst < out_next);
}
} while (in_next < in_fastloop_end && out_next < out_fastloop_end);
/*
* This is the generic loop for decoding literals and matches. This
* handles cases where in_next and out_next are close to the end of
* their respective buffers. Usually this loop isn't performance-
* critical, as most time is spent in the fastloop above instead. We
* therefore omit some optimizations here in favor of smaller code.
*/
generic_loop:
for (;;) {
u32 length, offset;
const u8 *src;
u8 *dst;
REFILL_BITS();
entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
saved_bitbuf = bitbuf;
bitbuf >>= (u8)entry;
bitsleft -= entry;
if (unlikely(entry & HUFFDEC_SUBTABLE_POINTER)) {
entry = d->u.litlen_decode_table[(entry >> 16) +
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
saved_bitbuf = bitbuf;
bitbuf >>= (u8)entry;
bitsleft -= entry;
}
length = entry >> 16;
if (entry & HUFFDEC_LITERAL) {
if (unlikely(out_next == out_end))
return LIBDEFLATE_INSUFFICIENT_SPACE;
*out_next++ = length;
continue;
}
if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
goto block_done;
length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
if (unlikely(length > out_end - out_next))
return LIBDEFLATE_INSUFFICIENT_SPACE;
if (!CAN_CONSUME(LENGTH_MAXBITS + OFFSET_MAXBITS))
REFILL_BITS();
entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
bitbuf >>= OFFSET_TABLEBITS;
bitsleft -= OFFSET_TABLEBITS;
entry = d->offset_decode_table[(entry >> 16) +
EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
if (!CAN_CONSUME(OFFSET_MAXBITS))
REFILL_BITS();
}
offset = entry >> 16;
offset += EXTRACT_VARBITS8(bitbuf, entry) >> (u8)(entry >> 8);
bitbuf >>= (u8)entry;
bitsleft -= entry;
SAFETY_CHECK(offset <= out_next - (const u8 *)out);
src = out_next - offset;
dst = out_next;
out_next += length;
STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
*dst++ = *src++;
*dst++ = *src++;
do {
*dst++ = *src++;
} while (dst < out_next);
}
block_done:
/* Finished decoding a block */
if (!is_final_block)
goto next_block;
/* That was the last block. */
bitsleft = (u8)bitsleft;
/*
* If any of the implicit appended zero bytes were consumed (not just
* refilled) before hitting end of stream, then the data is bad.
*/
SAFETY_CHECK(overread_count <= (bitsleft >> 3));
/* Optionally return the actual number of bytes consumed. */
if (actual_in_nbytes_ret) {
/* Don't count bytes that were refilled but not consumed. */
in_next -= (bitsleft >> 3) - overread_count;
*actual_in_nbytes_ret = in_next - (u8 *)in;
}
/* Optionally return the actual number of bytes written. */
if (actual_out_nbytes_ret) {
*actual_out_nbytes_ret = out_next - (u8 *)out;
} else {
if (out_next != out_end)
return LIBDEFLATE_SHORT_OUTPUT;
}
return LIBDEFLATE_SUCCESS;
}
#undef FUNCNAME
#undef ATTRIBUTES
#undef EXTRACT_VARBITS
#undef EXTRACT_VARBITS8
src/performance/libdeflate/lib/deflate_constants.h
+56
View File
@@ -0,0 +1,56 @@
/*
* deflate_constants.h - constants for the DEFLATE compression format
*/
#ifndef LIB_DEFLATE_CONSTANTS_H
#define LIB_DEFLATE_CONSTANTS_H
/* Valid block types */
#define DEFLATE_BLOCKTYPE_UNCOMPRESSED 0
#define DEFLATE_BLOCKTYPE_STATIC_HUFFMAN 1
#define DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN 2
/* Minimum and maximum supported match lengths (in bytes) */
#define DEFLATE_MIN_MATCH_LEN 3
#define DEFLATE_MAX_MATCH_LEN 258
/* Maximum supported match offset (in bytes) */
#define DEFLATE_MAX_MATCH_OFFSET 32768
/* log2 of DEFLATE_MAX_MATCH_OFFSET */
#define DEFLATE_WINDOW_ORDER 15
/* Number of symbols in each Huffman code. Note: for the literal/length
* and offset codes, these are actually the maximum values; a given block
* might use fewer symbols. */
#define DEFLATE_NUM_PRECODE_SYMS 19
#define DEFLATE_NUM_LITLEN_SYMS 288
#define DEFLATE_NUM_OFFSET_SYMS 32
/* The maximum number of symbols across all codes */
#define DEFLATE_MAX_NUM_SYMS 288
/* Division of symbols in the literal/length code */
#define DEFLATE_NUM_LITERALS 256
#define DEFLATE_END_OF_BLOCK 256
#define DEFLATE_FIRST_LEN_SYM 257
/* Maximum codeword length, in bits, within each Huffman code */
#define DEFLATE_MAX_PRE_CODEWORD_LEN 7
#define DEFLATE_MAX_LITLEN_CODEWORD_LEN 15
#define DEFLATE_MAX_OFFSET_CODEWORD_LEN 15
/* The maximum codeword length across all codes */
#define DEFLATE_MAX_CODEWORD_LEN 15
/* Maximum possible overrun when decoding codeword lengths */
#define DEFLATE_MAX_LENS_OVERRUN 137
/*
* Maximum number of extra bits that may be required to represent a match
* length or offset.
*/
#define DEFLATE_MAX_EXTRA_LENGTH_BITS 5
#define DEFLATE_MAX_EXTRA_OFFSET_BITS 13
#endif /* LIB_DEFLATE_CONSTANTS_H */
src/performance/libdeflate/lib/deflate_decompress.c
+1208
View File
File diff suppressed because it is too large Load Diff
src/performance/libdeflate/lib/lib_common.h
+106
View File
@@ -0,0 +1,106 @@
/*
* lib_common.h - internal header included by all library code
*/
#ifndef LIB_LIB_COMMON_H
#define LIB_LIB_COMMON_H
#ifdef LIBDEFLATE_H
/*
* When building the library, LIBDEFLATEAPI needs to be defined properly before
* including libdeflate.h.
*/
# error "lib_common.h must always be included before libdeflate.h"
#endif
#if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
# define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
#elif defined(__GNUC__)
# define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
#else
# define LIBDEFLATE_EXPORT_SYM
#endif
/*
* On i386, gcc assumes that the stack is 16-byte aligned at function entry.
* However, some compilers (e.g. MSVC) and programming languages (e.g. Delphi)
* only guarantee 4-byte alignment when calling functions. This is mainly an
* issue on Windows, but it has been seen on Linux too. Work around this ABI
* incompatibility by realigning the stack pointer when entering libdeflate.
* This prevents crashes in SSE/AVX code.
*/
#if defined(__GNUC__) && defined(__i386__)
# define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
#else
# define LIBDEFLATE_ALIGN_STACK
#endif
#define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
#include "../common_defs.h"
typedef void *(*malloc_func_t)(size_t);
typedef void (*free_func_t)(void *);
extern malloc_func_t libdeflate_default_malloc_func;
extern free_func_t libdeflate_default_free_func;
void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
size_t alignment, size_t size);
void libdeflate_aligned_free(free_func_t free_func, void *ptr);
#ifdef FREESTANDING
/*
* With -ffreestanding, <string.h> may be missing, and we must provide
* implementations of memset(), memcpy(), memmove(), and memcmp().
* See https://gcc.gnu.org/onlinedocs/gcc/Standards.html
*
* Also, -ffreestanding disables interpreting calls to these functions as
* built-ins. E.g., calling memcpy(&v, p, WORDBYTES) will make a function call,
* not be optimized to a single load instruction. For performance reasons we
* don't want that. So, declare these functions as macros that expand to the
* corresponding built-ins. This approach is recommended in the gcc man page.
* We still need the actual function definitions in case gcc calls them.
*/
void *memset(void *s, int c, size_t n);
#define memset(s, c, n) __builtin_memset((s), (c), (n))
void *memcpy(void *dest, const void *src, size_t n);
#define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
void *memmove(void *dest, const void *src, size_t n);
#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
int memcmp(const void *s1, const void *s2, size_t n);
#define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
#undef LIBDEFLATE_ENABLE_ASSERTIONS
#else
# include <string.h>
/*
* To prevent false positive static analyzer warnings, ensure that assertions
* are visible to the static analyzer.
*/
# ifdef __clang_analyzer__
# define LIBDEFLATE_ENABLE_ASSERTIONS
# endif
#endif
/*
* Runtime assertion support. Don't enable this in production builds; it may
* hurt performance significantly.
*/
#ifdef LIBDEFLATE_ENABLE_ASSERTIONS
NORETURN void
libdeflate_assertion_failed(const char *expr, const char *file, int line);
#define ASSERT(expr) { if (unlikely(!(expr))) \
libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
#else
#define ASSERT(expr) (void)(expr)
#endif
#define CONCAT_IMPL(a, b) a##b
#define CONCAT(a, b) CONCAT_IMPL(a, b)
#define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
#endif /* LIB_LIB_COMMON_H */
src/performance/libdeflate/lib/utils.c
+141
View File
@@ -0,0 +1,141 @@
/*
* utils.c - utility functions for libdeflate
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "lib_common.h"
#ifdef FREESTANDING
# define malloc NULL
# define free NULL
#else
# include <stdlib.h>
#endif
malloc_func_t libdeflate_default_malloc_func = malloc;
free_func_t libdeflate_default_free_func = free;
void *
libdeflate_aligned_malloc(malloc_func_t malloc_func,
size_t alignment, size_t size)
{
void *ptr = (*malloc_func)(sizeof(void *) + alignment - 1 + size);
if (ptr) {
void *orig_ptr = ptr;
ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment);
((void **)ptr)[-1] = orig_ptr;
}
return ptr;
}
void
libdeflate_aligned_free(free_func_t free_func, void *ptr)
{
(*free_func)(((void **)ptr)[-1]);
}
LIBDEFLATEAPI void
libdeflate_set_memory_allocator(malloc_func_t malloc_func,
free_func_t free_func)
{
libdeflate_default_malloc_func = malloc_func;
libdeflate_default_free_func = free_func;
}
/*
* Implementations of libc functions for freestanding library builds.
* Normal library builds don't use these. Not optimized yet; usually the
* compiler expands these functions and doesn't actually call them anyway.
*/
#ifdef FREESTANDING
#undef memset
void * __attribute__((weak))
memset(void *s, int c, size_t n)
{
u8 *p = s;
size_t i;
for (i = 0; i < n; i++)
p[i] = c;
return s;
}
#undef memcpy
void * __attribute__((weak))
memcpy(void *dest, const void *src, size_t n)
{
u8 *d = dest;
const u8 *s = src;
size_t i;
for (i = 0; i < n; i++)
d[i] = s[i];
return dest;
}
#undef memmove
void * __attribute__((weak))
memmove(void *dest, const void *src, size_t n)
{
u8 *d = dest;
const u8 *s = src;
size_t i;
if (d <= s)
return memcpy(d, s, n);
for (i = n; i > 0; i--)
d[i - 1] = s[i - 1];
return dest;
}
#undef memcmp
int __attribute__((weak))
memcmp(const void *s1, const void *s2, size_t n)
{
const u8 *p1 = s1;
const u8 *p2 = s2;
size_t i;
for (i = 0; i < n; i++) {
if (p1[i] != p2[i])
return (int)p1[i] - (int)p2[i];
}
return 0;
}
#endif /* FREESTANDING */
#ifdef LIBDEFLATE_ENABLE_ASSERTIONS
#include <stdio.h>
#include <stdlib.h>
NORETURN void
libdeflate_assertion_failed(const char *expr, const char *file, int line)
{
fprintf(stderr, "Assertion failed: %s at %s:%d\n", expr, file, line);
abort();
}
#endif /* LIBDEFLATE_ENABLE_ASSERTIONS */
src/performance/libdeflate/lib/x86/adler32_impl.h
+135
View File
@@ -0,0 +1,135 @@
/*
* x86/adler32_impl.h - x86 implementations of Adler-32 checksum algorithm
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef LIB_X86_ADLER32_IMPL_H
#define LIB_X86_ADLER32_IMPL_H
#include "cpu_features.h"
/* SSE2 and AVX2 implementations. Used on older CPUs. */
#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
# define adler32_x86_sse2 adler32_x86_sse2
# define SUFFIX _sse2
# define ATTRIBUTES _target_attribute("sse2")
# define VL 16
# define USE_VNNI 0
# define USE_AVX512 0
# include "adler32_template.h"
# define adler32_x86_avx2 adler32_x86_avx2
# define SUFFIX _avx2
# define ATTRIBUTES _target_attribute("avx2")
# define VL 32
# define USE_VNNI 0
# define USE_AVX512 0
# include "adler32_template.h"
#endif
/*
* AVX-VNNI implementation. This is used on CPUs that have AVX2 and AVX-VNNI
* but don't have AVX-512, for example Intel Alder Lake.
*
* Unusually for a new CPU feature, gcc added support for the AVX-VNNI
* intrinsics (in gcc 11.1) slightly before binutils added support for
* assembling AVX-VNNI instructions (in binutils 2.36). Distros can reasonably
* have gcc 11 with binutils 2.35. Because of this issue, we check for gcc 12
* instead of gcc 11. (libdeflate supports direct compilation without a
* configure step, so checking the binutils version is not always an option.)
*/
#if (GCC_PREREQ(12, 1) || CLANG_PREREQ(12, 0, 13000000) || MSVC_PREREQ(1930)) && \
!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX_VNNI)
# define adler32_x86_avx2_vnni adler32_x86_avx2_vnni
# define SUFFIX _avx2_vnni
# define ATTRIBUTES _target_attribute("avx2,avxvnni")
# define VL 32
# define USE_VNNI 1
# define USE_AVX512 0
# include "adler32_template.h"
#endif
#if (GCC_PREREQ(8, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX512VNNI)
/*
* AVX512VNNI implementation using 256-bit vectors. This is very similar to the
* AVX-VNNI implementation but takes advantage of masking and more registers.
* This is used on certain older Intel CPUs, specifically Ice Lake and Tiger
* Lake, which support AVX512VNNI but downclock a bit too eagerly when ZMM
* registers are used.
*/
# define adler32_x86_avx512_vl256_vnni adler32_x86_avx512_vl256_vnni
# define SUFFIX _avx512_vl256_vnni
# define ATTRIBUTES _target_attribute("avx512bw,avx512vl,avx512vnni")
# define VL 32
# define USE_VNNI 1
# define USE_AVX512 1
# include "adler32_template.h"
/*
* AVX512VNNI implementation using 512-bit vectors. This is used on CPUs that
* have a good AVX-512 implementation including AVX512VNNI.
*/
# define adler32_x86_avx512_vl512_vnni adler32_x86_avx512_vl512_vnni
# define SUFFIX _avx512_vl512_vnni
# define ATTRIBUTES _target_attribute("avx512bw,avx512vnni")
# define VL 64
# define USE_VNNI 1
# define USE_AVX512 1
# include "adler32_template.h"
#endif
static inline adler32_func_t
arch_select_adler32_func(void)
{
const u32 features MAYBE_UNUSED = get_x86_cpu_features();
#ifdef adler32_x86_avx512_vl512_vnni
if ((features & X86_CPU_FEATURE_ZMM) &&
HAVE_AVX512BW(features) && HAVE_AVX512VNNI(features))
return adler32_x86_avx512_vl512_vnni;
#endif
#ifdef adler32_x86_avx512_vl256_vnni
if (HAVE_AVX512BW(features) && HAVE_AVX512VL(features) &&
HAVE_AVX512VNNI(features))
return adler32_x86_avx512_vl256_vnni;
#endif
#ifdef adler32_x86_avx2_vnni
if (HAVE_AVX2(features) && HAVE_AVXVNNI(features))
return adler32_x86_avx2_vnni;
#endif
#ifdef adler32_x86_avx2
if (HAVE_AVX2(features))
return adler32_x86_avx2;
#endif
#ifdef adler32_x86_sse2
if (HAVE_SSE2(features))
return adler32_x86_sse2;
#endif
return NULL;
}
#define arch_select_adler32_func arch_select_adler32_func
#endif /* LIB_X86_ADLER32_IMPL_H */
src/performance/libdeflate/lib/x86/adler32_template.h
+518
View File
@@ -0,0 +1,518 @@
/*
* x86/adler32_template.h - template for vectorized Adler-32 implementations
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
/*
* This file is a "template" for instantiating Adler-32 functions for x86.
* The "parameters" are:
*
* SUFFIX:
* Name suffix to append to all instantiated functions.
* ATTRIBUTES:
* Target function attributes to use. Must satisfy the dependencies of the
* other parameters as follows:
* VL=16 && USE_VNNI=0 && USE_AVX512=0: at least sse2
* VL=32 && USE_VNNI=0 && USE_AVX512=0: at least avx2
* VL=32 && USE_VNNI=1 && USE_AVX512=0: at least avx2,avxvnni
* VL=32 && USE_VNNI=1 && USE_AVX512=1: at least avx512bw,avx512vl,avx512vnni
* VL=64 && USE_VNNI=1 && USE_AVX512=1: at least avx512bw,avx512vnni
* (Other combinations are not useful and have not been tested.)
* VL:
* Vector length in bytes. Must be 16, 32, or 64.
* USE_VNNI:
* If 1, use the VNNI dot product based algorithm.
* If 0, use the legacy SSE2 and AVX2 compatible algorithm.
* USE_AVX512:
* If 1, take advantage of AVX-512 features such as masking. This doesn't
* enable the use of 512-bit vectors; the vector length is controlled by
* VL. If 0, assume that the CPU might not support AVX-512.
*/
#if VL == 16
# define vec_t __m128i
# define mask_t u16
# define LOG2_VL 4
# define VADD8(a, b) _mm_add_epi8((a), (b))
# define VADD16(a, b) _mm_add_epi16((a), (b))
# define VADD32(a, b) _mm_add_epi32((a), (b))
# if USE_AVX512
# define VDPBUSD(a, b, c) _mm_dpbusd_epi32((a), (b), (c))
# else
# define VDPBUSD(a, b, c) _mm_dpbusd_avx_epi32((a), (b), (c))
# endif
# define VLOAD(p) _mm_load_si128((const void *)(p))
# define VLOADU(p) _mm_loadu_si128((const void *)(p))
# define VMADD16(a, b) _mm_madd_epi16((a), (b))
# define VMASKZ_LOADU(mask, p) _mm_maskz_loadu_epi8((mask), (p))
# define VMULLO32(a, b) _mm_mullo_epi32((a), (b))
# define VSAD8(a, b) _mm_sad_epu8((a), (b))
# define VSET1_8(a) _mm_set1_epi8(a)
# define VSET1_32(a) _mm_set1_epi32(a)
# define VSETZERO() _mm_setzero_si128()
# define VSLL32(a, b) _mm_slli_epi32((a), (b))
# define VUNPACKLO8(a, b) _mm_unpacklo_epi8((a), (b))
# define VUNPACKHI8(a, b) _mm_unpackhi_epi8((a), (b))
#elif VL == 32
# define vec_t __m256i
# define mask_t u32
# define LOG2_VL 5
# define VADD8(a, b) _mm256_add_epi8((a), (b))
# define VADD16(a, b) _mm256_add_epi16((a), (b))
# define VADD32(a, b) _mm256_add_epi32((a), (b))
# if USE_AVX512
# define VDPBUSD(a, b, c) _mm256_dpbusd_epi32((a), (b), (c))
# else
# define VDPBUSD(a, b, c) _mm256_dpbusd_avx_epi32((a), (b), (c))
# endif
# define VLOAD(p) _mm256_load_si256((const void *)(p))
# define VLOADU(p) _mm256_loadu_si256((const void *)(p))
# define VMADD16(a, b) _mm256_madd_epi16((a), (b))
# define VMASKZ_LOADU(mask, p) _mm256_maskz_loadu_epi8((mask), (p))
# define VMULLO32(a, b) _mm256_mullo_epi32((a), (b))
# define VSAD8(a, b) _mm256_sad_epu8((a), (b))
# define VSET1_8(a) _mm256_set1_epi8(a)
# define VSET1_32(a) _mm256_set1_epi32(a)
# define VSETZERO() _mm256_setzero_si256()
# define VSLL32(a, b) _mm256_slli_epi32((a), (b))
# define VUNPACKLO8(a, b) _mm256_unpacklo_epi8((a), (b))
# define VUNPACKHI8(a, b) _mm256_unpackhi_epi8((a), (b))
#elif VL == 64
# define vec_t __m512i
# define mask_t u64
# define LOG2_VL 6
# define VADD8(a, b) _mm512_add_epi8((a), (b))
# define VADD16(a, b) _mm512_add_epi16((a), (b))
# define VADD32(a, b) _mm512_add_epi32((a), (b))
# define VDPBUSD(a, b, c) _mm512_dpbusd_epi32((a), (b), (c))
# define VLOAD(p) _mm512_load_si512((const void *)(p))
# define VLOADU(p) _mm512_loadu_si512((const void *)(p))
# define VMADD16(a, b) _mm512_madd_epi16((a), (b))
# define VMASKZ_LOADU(mask, p) _mm512_maskz_loadu_epi8((mask), (p))
# define VMULLO32(a, b) _mm512_mullo_epi32((a), (b))
# define VSAD8(a, b) _mm512_sad_epu8((a), (b))
# define VSET1_8(a) _mm512_set1_epi8(a)
# define VSET1_32(a) _mm512_set1_epi32(a)
# define VSETZERO() _mm512_setzero_si512()
# define VSLL32(a, b) _mm512_slli_epi32((a), (b))
# define VUNPACKLO8(a, b) _mm512_unpacklo_epi8((a), (b))
# define VUNPACKHI8(a, b) _mm512_unpackhi_epi8((a), (b))
#else
# error "unsupported vector length"
#endif
#define VADD32_3X(a, b, c) VADD32(VADD32((a), (b)), (c))
#define VADD32_4X(a, b, c, d) VADD32(VADD32((a), (b)), VADD32((c), (d)))
#define VADD32_5X(a, b, c, d, e) VADD32((a), VADD32_4X((b), (c), (d), (e)))
#define VADD32_7X(a, b, c, d, e, f, g) \
VADD32(VADD32_3X((a), (b), (c)), VADD32_4X((d), (e), (f), (g)))
/* Sum the 32-bit elements of v_s1 and add them to s1, and likewise for s2. */
#undef reduce_to_32bits
static forceinline ATTRIBUTES void
ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
{
__m128i v_s1_128, v_s2_128;
#if VL == 16
{
v_s1_128 = v_s1;
v_s2_128 = v_s2;
}
#else
{
__m256i v_s1_256, v_s2_256;
#if VL == 32
v_s1_256 = v_s1;
v_s2_256 = v_s2;
#else
/* Reduce 512 bits to 256 bits. */
v_s1_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s1, 0),
_mm512_extracti64x4_epi64(v_s1, 1));
v_s2_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s2, 0),
_mm512_extracti64x4_epi64(v_s2, 1));
#endif
/* Reduce 256 bits to 128 bits. */
v_s1_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s1_256, 0),
_mm256_extracti128_si256(v_s1_256, 1));
v_s2_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s2_256, 0),
_mm256_extracti128_si256(v_s2_256, 1));
}
#endif
/*
* Reduce 128 bits to 32 bits.
*
* If the bytes were summed into v_s1 using psadbw + paddd, then ignore
* the odd-indexed elements of v_s1_128 since they are zero.
*/
#if USE_VNNI
v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x31));
#endif
v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x31));
v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x02));
v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x02));
*s1_p += (u32)_mm_cvtsi128_si32(v_s1_128);
*s2_p += (u32)_mm_cvtsi128_si32(v_s2_128);
}
#define reduce_to_32bits ADD_SUFFIX(reduce_to_32bits)
static ATTRIBUTES u32
ADD_SUFFIX(adler32_x86)(u32 adler, const u8 *p, size_t len)
{
#if USE_VNNI
/* This contains the bytes [VL, VL-1, VL-2, ..., 1]. */
static const u8 _aligned_attribute(VL) raw_mults[VL] = {
#if VL == 64
64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
#endif
#if VL >= 32
32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
#endif
16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
};
const vec_t ones = VSET1_8(1);
#else
/*
* This contains the 16-bit values [2*VL, 2*VL - 1, 2*VL - 2, ..., 1].
* For VL==32 the ordering is weird because it has to match the way that
* vpunpcklbw and vpunpckhbw work on 128-bit lanes separately.
*/
static const u16 _aligned_attribute(VL) raw_mults[4][VL / 2] = {
#if VL == 16
{ 32, 31, 30, 29, 28, 27, 26, 25 },
{ 24, 23, 22, 21, 20, 19, 18, 17 },
{ 16, 15, 14, 13, 12, 11, 10, 9 },
{ 8, 7, 6, 5, 4, 3, 2, 1 },
#elif VL == 32
{ 64, 63, 62, 61, 60, 59, 58, 57, 48, 47, 46, 45, 44, 43, 42, 41 },
{ 56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33 },
{ 32, 31, 30, 29, 28, 27, 26, 25, 16, 15, 14, 13, 12, 11, 10, 9 },
{ 24, 23, 22, 21, 20, 19, 18, 17, 8, 7, 6, 5, 4, 3, 2, 1 },
#else
# error "unsupported parameters"
#endif
};
const vec_t mults_a = VLOAD(raw_mults[0]);
const vec_t mults_b = VLOAD(raw_mults[1]);
const vec_t mults_c = VLOAD(raw_mults[2]);
const vec_t mults_d = VLOAD(raw_mults[3]);
#endif
const vec_t zeroes = VSETZERO();
u32 s1 = adler & 0xFFFF;
u32 s2 = adler >> 16;
/*
* If the length is large and the pointer is misaligned, align it.
* For smaller lengths, just take the misaligned load penalty.
*/
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) {
do {
s1 += *p++;
s2 += s1;
len--;
} while ((uintptr_t)p & (VL-1));
s1 %= DIVISOR;
s2 %= DIVISOR;
}
#if USE_VNNI
/*
* This is Adler-32 using the vpdpbusd instruction from AVX512VNNI or
* AVX-VNNI. vpdpbusd multiplies the unsigned bytes of one vector by
* the signed bytes of another vector and adds the sums in groups of 4
* to the 32-bit elements of a third vector. We use it in two ways:
* multiplying the data bytes by a sequence like 64,63,62,...,1 for
* calculating part of s2, and multiplying the data bytes by an all-ones
* sequence 1,1,1,...,1 for calculating s1 and part of s2. The all-ones
* trick seems to be faster than the alternative of vpsadbw + vpaddd.
*/
while (len) {
/*
* Calculate the length of the next data chunk such that s1 and
* s2 are guaranteed to not exceed UINT32_MAX.
*/
size_t n = MIN(len, MAX_CHUNK_LEN & ~(4*VL - 1));
vec_t mults = VLOAD(raw_mults);
vec_t v_s1 = zeroes;
vec_t v_s2 = zeroes;
s2 += s1 * n;
len -= n;
if (n >= 4*VL) {
vec_t v_s1_b = zeroes;
vec_t v_s1_c = zeroes;
vec_t v_s1_d = zeroes;
vec_t v_s2_b = zeroes;
vec_t v_s2_c = zeroes;
vec_t v_s2_d = zeroes;
vec_t v_s1_sums = zeroes;
vec_t v_s1_sums_b = zeroes;
vec_t v_s1_sums_c = zeroes;
vec_t v_s1_sums_d = zeroes;
vec_t tmp0, tmp1;
do {
vec_t data_a = VLOADU(p + 0*VL);
vec_t data_b = VLOADU(p + 1*VL);
vec_t data_c = VLOADU(p + 2*VL);
vec_t data_d = VLOADU(p + 3*VL);
/*
* Workaround for gcc bug where it generates
* unnecessary move instructions
* (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107892)
*/
#if GCC_PREREQ(1, 0)
__asm__("" : "+v" (data_a), "+v" (data_b),
"+v" (data_c), "+v" (data_d));
#endif
v_s2 = VDPBUSD(v_s2, data_a, mults);
v_s2_b = VDPBUSD(v_s2_b, data_b, mults);
v_s2_c = VDPBUSD(v_s2_c, data_c, mults);
v_s2_d = VDPBUSD(v_s2_d, data_d, mults);
v_s1_sums = VADD32(v_s1_sums, v_s1);
v_s1_sums_b = VADD32(v_s1_sums_b, v_s1_b);
v_s1_sums_c = VADD32(v_s1_sums_c, v_s1_c);
v_s1_sums_d = VADD32(v_s1_sums_d, v_s1_d);
v_s1 = VDPBUSD(v_s1, data_a, ones);
v_s1_b = VDPBUSD(v_s1_b, data_b, ones);
v_s1_c = VDPBUSD(v_s1_c, data_c, ones);
v_s1_d = VDPBUSD(v_s1_d, data_d, ones);
/* Same gcc bug workaround. See above */
#if GCC_PREREQ(1, 0) && !defined(ARCH_X86_32)
__asm__("" : "+v" (v_s2), "+v" (v_s2_b),
"+v" (v_s2_c), "+v" (v_s2_d),
"+v" (v_s1_sums),
"+v" (v_s1_sums_b),
"+v" (v_s1_sums_c),
"+v" (v_s1_sums_d),
"+v" (v_s1), "+v" (v_s1_b),
"+v" (v_s1_c), "+v" (v_s1_d));
#endif
p += 4*VL;
n -= 4*VL;
} while (n >= 4*VL);
/*
* Reduce into v_s1 and v_s2 as follows:
*
* v_s2 = v_s2 + v_s2_b + v_s2_c + v_s2_d +
* (4*VL)*(v_s1_sums + v_s1_sums_b +
* v_s1_sums_c + v_s1_sums_d) +
* (3*VL)*v_s1 + (2*VL)*v_s1_b + VL*v_s1_c
* v_s1 = v_s1 + v_s1_b + v_s1_c + v_s1_d
*/
tmp0 = VADD32(v_s1, v_s1_b);
tmp1 = VADD32(v_s1, v_s1_c);
v_s1_sums = VADD32_4X(v_s1_sums, v_s1_sums_b,
v_s1_sums_c, v_s1_sums_d);
v_s1 = VADD32_3X(tmp0, v_s1_c, v_s1_d);
v_s2 = VADD32_7X(VSLL32(v_s1_sums, LOG2_VL + 2),
VSLL32(tmp0, LOG2_VL + 1),
VSLL32(tmp1, LOG2_VL),
v_s2, v_s2_b, v_s2_c, v_s2_d);
}
/* Process the last 0 <= n < 4*VL bytes of the chunk. */
if (n >= 2*VL) {
const vec_t data_a = VLOADU(p + 0*VL);
const vec_t data_b = VLOADU(p + 1*VL);
v_s2 = VADD32(v_s2, VSLL32(v_s1, LOG2_VL + 1));
v_s1 = VDPBUSD(v_s1, data_a, ones);
v_s1 = VDPBUSD(v_s1, data_b, ones);
v_s2 = VDPBUSD(v_s2, data_a, VSET1_8(VL));
v_s2 = VDPBUSD(v_s2, data_a, mults);
v_s2 = VDPBUSD(v_s2, data_b, mults);
p += 2*VL;
n -= 2*VL;
}
if (n) {
/* Process the last 0 < n < 2*VL bytes of the chunk. */
vec_t data;
v_s2 = VADD32(v_s2, VMULLO32(v_s1, VSET1_32(n)));
mults = VADD8(mults, VSET1_8((int)n - VL));
if (n > VL) {
data = VLOADU(p);
v_s1 = VDPBUSD(v_s1, data, ones);
v_s2 = VDPBUSD(v_s2, data, mults);
p += VL;
n -= VL;
mults = VADD8(mults, VSET1_8(-VL));
}
/*
* Process the last 0 < n <= VL bytes of the chunk.
* Utilize a masked load if it's available.
*/
#if USE_AVX512
data = VMASKZ_LOADU((mask_t)-1 >> (VL - n), p);
#else
data = zeroes;
memcpy(&data, p, n);
#endif
v_s1 = VDPBUSD(v_s1, data, ones);
v_s2 = VDPBUSD(v_s2, data, mults);
p += n;
}
reduce_to_32bits(v_s1, v_s2, &s1, &s2);
s1 %= DIVISOR;
s2 %= DIVISOR;
}
#else /* USE_VNNI */
/*
* This is Adler-32 for SSE2 and AVX2.
*
* To horizontally sum bytes, use psadbw + paddd, where one of the
* arguments to psadbw is all-zeroes.
*
* For the s2 contribution from (2*VL - i)*data[i] for each of the 2*VL
* bytes of each iteration of the inner loop, use punpck{l,h}bw + paddw
* to sum, for each i across iterations, byte i into a corresponding
* 16-bit counter in v_byte_sums_*. After the inner loop, use pmaddwd
* to multiply each counter by (2*VL - i), then add the products to s2.
*
* An alternative implementation would use pmaddubsw and pmaddwd in the
* inner loop to do (2*VL - i)*data[i] directly and add the products in
* groups of 4 to 32-bit counters. However, on average that approach
* seems to be slower than the current approach which delays the
* multiplications. Also, pmaddubsw requires SSSE3; the current
* approach keeps the implementation aligned between SSE2 and AVX2.
*
* The inner loop processes 2*VL bytes per iteration. Increasing this
* to 4*VL doesn't seem to be helpful here.
*/
while (len) {
/*
* Calculate the length of the next data chunk such that s1 and
* s2 are guaranteed to not exceed UINT32_MAX, and every
* v_byte_sums_* counter is guaranteed to not exceed INT16_MAX.
* It's INT16_MAX, not UINT16_MAX, because v_byte_sums_* are
* used with pmaddwd which does signed multiplication. In the
* SSE2 case this limits chunks to 4096 bytes instead of 5536.
*/
size_t n = MIN(len, MIN(2 * VL * (INT16_MAX / UINT8_MAX),
MAX_CHUNK_LEN) & ~(2*VL - 1));
len -= n;
if (n >= 2*VL) {
vec_t v_s1 = zeroes;
vec_t v_s1_sums = zeroes;
vec_t v_byte_sums_a = zeroes;
vec_t v_byte_sums_b = zeroes;
vec_t v_byte_sums_c = zeroes;
vec_t v_byte_sums_d = zeroes;
vec_t v_s2;
s2 += s1 * (n & ~(2*VL - 1));
do {
vec_t data_a = VLOADU(p + 0*VL);
vec_t data_b = VLOADU(p + 1*VL);
v_s1_sums = VADD32(v_s1_sums, v_s1);
v_byte_sums_a = VADD16(v_byte_sums_a,
VUNPACKLO8(data_a, zeroes));
v_byte_sums_b = VADD16(v_byte_sums_b,
VUNPACKHI8(data_a, zeroes));
v_byte_sums_c = VADD16(v_byte_sums_c,
VUNPACKLO8(data_b, zeroes));
v_byte_sums_d = VADD16(v_byte_sums_d,
VUNPACKHI8(data_b, zeroes));
v_s1 = VADD32(v_s1,
VADD32(VSAD8(data_a, zeroes),
VSAD8(data_b, zeroes)));
/*
* Workaround for gcc bug where it generates
* unnecessary move instructions
* (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107892)
*/
#if GCC_PREREQ(1, 0)
__asm__("" : "+x" (v_s1), "+x" (v_s1_sums),
"+x" (v_byte_sums_a),
"+x" (v_byte_sums_b),
"+x" (v_byte_sums_c),
"+x" (v_byte_sums_d));
#endif
p += 2*VL;
n -= 2*VL;
} while (n >= 2*VL);
/*
* Calculate v_s2 as (2*VL)*v_s1_sums +
* [2*VL, 2*VL - 1, 2*VL - 2, ..., 1] * v_byte_sums.
* Then update s1 and s2 from v_s1 and v_s2.
*/
v_s2 = VADD32_5X(VSLL32(v_s1_sums, LOG2_VL + 1),
VMADD16(v_byte_sums_a, mults_a),
VMADD16(v_byte_sums_b, mults_b),
VMADD16(v_byte_sums_c, mults_c),
VMADD16(v_byte_sums_d, mults_d));
reduce_to_32bits(v_s1, v_s2, &s1, &s2);
}
/*
* Process the last 0 <= n < 2*VL bytes of the chunk using
* scalar instructions and reduce s1 and s2 mod DIVISOR.
*/
ADLER32_CHUNK(s1, s2, p, n);
}
#endif /* !USE_VNNI */
return (s2 << 16) | s1;
}
#undef vec_t
#undef mask_t
#undef LOG2_VL
#undef VADD8
#undef VADD16
#undef VADD32
#undef VDPBUSD
#undef VLOAD
#undef VLOADU
#undef VMADD16
#undef VMASKZ_LOADU
#undef VMULLO32
#undef VSAD8
#undef VSET1_8
#undef VSET1_32
#undef VSETZERO
#undef VSLL32
#undef VUNPACKLO8
#undef VUNPACKHI8
#undef SUFFIX
#undef ATTRIBUTES
#undef VL
#undef USE_VNNI
#undef USE_AVX512
src/performance/libdeflate/lib/x86/cpu_features.c
+213
View File
@@ -0,0 +1,213 @@
/*
* x86/cpu_features.c - feature detection for x86 CPUs
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "../cpu_features_common.h" /* must be included first */
#include "cpu_features.h"
#ifdef X86_CPU_FEATURES_KNOWN
/* Runtime x86 CPU feature detection is supported. */
/* Execute the CPUID instruction. */
static inline void
cpuid(u32 leaf, u32 subleaf, u32 *a, u32 *b, u32 *c, u32 *d)
{
#ifdef _MSC_VER
int result[4];
__cpuidex(result, leaf, subleaf);
*a = result[0];
*b = result[1];
*c = result[2];
*d = result[3];
#else
__asm__ volatile("cpuid" : "=a" (*a), "=b" (*b), "=c" (*c), "=d" (*d)
: "a" (leaf), "c" (subleaf));
#endif
}
/* Read an extended control register. */
static inline u64
read_xcr(u32 index)
{
#ifdef _MSC_VER
return _xgetbv(index);
#else
u32 d, a;
/*
* Execute the "xgetbv" instruction. Old versions of binutils do not
* recognize this instruction, so list the raw bytes instead.
*
* This must be 'volatile' to prevent this code from being moved out
* from under the check for OSXSAVE.
*/
__asm__ volatile(".byte 0x0f, 0x01, 0xd0" :
"=d" (d), "=a" (a) : "c" (index));
return ((u64)d << 32) | a;
#endif
}
static const struct cpu_feature x86_cpu_feature_table[] = {
{X86_CPU_FEATURE_SSE2, "sse2"},
{X86_CPU_FEATURE_PCLMULQDQ, "pclmulqdq"},
{X86_CPU_FEATURE_AVX, "avx"},
{X86_CPU_FEATURE_AVX2, "avx2"},
{X86_CPU_FEATURE_BMI2, "bmi2"},
{X86_CPU_FEATURE_ZMM, "zmm"},
{X86_CPU_FEATURE_AVX512BW, "avx512bw"},
{X86_CPU_FEATURE_AVX512VL, "avx512vl"},
{X86_CPU_FEATURE_VPCLMULQDQ, "vpclmulqdq"},
{X86_CPU_FEATURE_AVX512VNNI, "avx512_vnni"},
{X86_CPU_FEATURE_AVXVNNI, "avx_vnni"},
};
volatile u32 libdeflate_x86_cpu_features = 0;
static inline bool
os_supports_avx512(u64 xcr0)
{
#ifdef __APPLE__
/*
* The Darwin kernel had a bug where it could corrupt the opmask
* registers. See
* https://community.intel.com/t5/Software-Tuning-Performance/MacOS-Darwin-kernel-bug-clobbers-AVX-512-opmask-register-state/m-p/1327259
* Darwin also does not initially set the XCR0 bits for AVX512, but they
* are set if the thread tries to use AVX512 anyway. Thus, to safely
* and consistently use AVX512 on macOS we'd need to check the kernel
* version as well as detect AVX512 support using a macOS-specific
* method. We don't bother with this, especially given Apple's
* transition to arm64.
*/
return false;
#else
return (xcr0 & 0xe6) == 0xe6;
#endif
}
/*
* Don't use 512-bit vectors (ZMM registers) on Intel CPUs before Rocket Lake
* and Sapphire Rapids, due to the overly-eager downclocking which can reduce
* the performance of workloads that use ZMM registers only occasionally.
*/
static inline bool
allow_512bit_vectors(const u32 manufacturer[3], u32 family, u32 model)
{
#ifdef TEST_SUPPORT__DO_NOT_USE
return true;
#endif
if (memcmp(manufacturer, "GenuineIntel", 12) != 0)
return true;
if (family != 6)
return true;
switch (model) {
case 85: /* Skylake (Server), Cascade Lake, Cooper Lake */
case 106: /* Ice Lake (Server) */
case 108: /* Ice Lake (Server) */
case 126: /* Ice Lake (Client) */
case 140: /* Tiger Lake */
case 141: /* Tiger Lake */
return false;
}
return true;
}
/* Initialize libdeflate_x86_cpu_features. */
void libdeflate_init_x86_cpu_features(void)
{
u32 max_leaf;
u32 manufacturer[3];
u32 family, model;
u32 a, b, c, d;
u64 xcr0 = 0;
u32 features = 0;
/* EAX=0: Highest Function Parameter and Manufacturer ID */
cpuid(0, 0, &max_leaf, &manufacturer[0], &manufacturer[2],
&manufacturer[1]);
if (max_leaf < 1)
goto out;
/* EAX=1: Processor Info and Feature Bits */
cpuid(1, 0, &a, &b, &c, &d);
family = (a >> 8) & 0xf;
model = (a >> 4) & 0xf;
if (family == 6 || family == 0xf)
model += (a >> 12) & 0xf0;
if (family == 0xf)
family += (a >> 20) & 0xff;
if (d & (1 << 26))
features |= X86_CPU_FEATURE_SSE2;
/*
* No known CPUs have pclmulqdq without sse4.1, so in practice code
* targeting pclmulqdq can use sse4.1 instructions. But to be safe,
* explicitly check for both the pclmulqdq and sse4.1 bits.
*/
if ((c & (1 << 1)) && (c & (1 << 19)))
features |= X86_CPU_FEATURE_PCLMULQDQ;
if (c & (1 << 27))
xcr0 = read_xcr(0);
if ((c & (1 << 28)) && ((xcr0 & 0x6) == 0x6))
features |= X86_CPU_FEATURE_AVX;
if (max_leaf < 7)
goto out;
/* EAX=7, ECX=0: Extended Features */
cpuid(7, 0, &a, &b, &c, &d);
if (b & (1 << 8))
features |= X86_CPU_FEATURE_BMI2;
if ((xcr0 & 0x6) == 0x6) {
if (b & (1 << 5))
features |= X86_CPU_FEATURE_AVX2;
if (c & (1 << 10))
features |= X86_CPU_FEATURE_VPCLMULQDQ;
}
if (os_supports_avx512(xcr0)) {
if (allow_512bit_vectors(manufacturer, family, model))
features |= X86_CPU_FEATURE_ZMM;
if (b & (1 << 30))
features |= X86_CPU_FEATURE_AVX512BW;
if (b & (1U << 31))
features |= X86_CPU_FEATURE_AVX512VL;
if (c & (1 << 11))
features |= X86_CPU_FEATURE_AVX512VNNI;
}
/* EAX=7, ECX=1: Extended Features */
cpuid(7, 1, &a, &b, &c, &d);
if ((a & (1 << 4)) && ((xcr0 & 0x6) == 0x6))
features |= X86_CPU_FEATURE_AVXVNNI;
out:
disable_cpu_features_for_testing(&features, x86_cpu_feature_table,
ARRAY_LEN(x86_cpu_feature_table));
libdeflate_x86_cpu_features = features | X86_CPU_FEATURES_KNOWN;
}
#endif /* X86_CPU_FEATURES_KNOWN */
src/performance/libdeflate/lib/x86/cpu_features.h
+170
View File
@@ -0,0 +1,170 @@
/*
* x86/cpu_features.h - feature detection for x86 CPUs
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef LIB_X86_CPU_FEATURES_H
#define LIB_X86_CPU_FEATURES_H
#include "../lib_common.h"
#if defined(ARCH_X86_32) || defined(ARCH_X86_64)
#define X86_CPU_FEATURE_SSE2 (1 << 0)
#define X86_CPU_FEATURE_PCLMULQDQ (1 << 1)
#define X86_CPU_FEATURE_AVX (1 << 2)
#define X86_CPU_FEATURE_AVX2 (1 << 3)
#define X86_CPU_FEATURE_BMI2 (1 << 4)
/*
* ZMM indicates whether 512-bit vectors (zmm registers) should be used. On
* some CPUs, to avoid downclocking issues we don't set ZMM even if the CPU and
* operating system support AVX-512. On these CPUs, we may still use AVX-512
* instructions, but only with xmm and ymm registers.
*/
#define X86_CPU_FEATURE_ZMM (1 << 5)
#define X86_CPU_FEATURE_AVX512BW (1 << 6)
#define X86_CPU_FEATURE_AVX512VL (1 << 7)
#define X86_CPU_FEATURE_VPCLMULQDQ (1 << 8)
#define X86_CPU_FEATURE_AVX512VNNI (1 << 9)
#define X86_CPU_FEATURE_AVXVNNI (1 << 10)
#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
/* Runtime x86 CPU feature detection is supported. */
# define X86_CPU_FEATURES_KNOWN (1U << 31)
extern volatile u32 libdeflate_x86_cpu_features;
void libdeflate_init_x86_cpu_features(void);
static inline u32 get_x86_cpu_features(void)
{
if (libdeflate_x86_cpu_features == 0)
libdeflate_init_x86_cpu_features();
return libdeflate_x86_cpu_features;
}
/*
* x86 intrinsics are also supported. Include the headers needed to use them.
* Normally just immintrin.h suffices. With clang in MSVC compatibility mode,
* immintrin.h incorrectly skips including sub-headers, so include those too.
*/
# include <immintrin.h>
# if defined(_MSC_VER) && defined(__clang__)
# include <tmmintrin.h>
# include <smmintrin.h>
# include <wmmintrin.h>
# include <avxintrin.h>
# include <avx2intrin.h>
# include <avx512fintrin.h>
# include <avx512bwintrin.h>
# include <avx512vlintrin.h>
# if __has_include(<avx512vlbwintrin.h>)
# include <avx512vlbwintrin.h>
# endif
# if __has_include(<vpclmulqdqintrin.h>)
# include <vpclmulqdqintrin.h>
# endif
# if __has_include(<avx512vnniintrin.h>)
# include <avx512vnniintrin.h>
# endif
# if __has_include(<avx512vlvnniintrin.h>)
# include <avx512vlvnniintrin.h>
# endif
# if __has_include(<avxvnniintrin.h>)
# include <avxvnniintrin.h>
# endif
# endif
#else
static inline u32 get_x86_cpu_features(void) { return 0; }
#endif
#if defined(__SSE2__) || \
(defined(_MSC_VER) && \
(defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
# define HAVE_SSE2(features) 1
# define HAVE_SSE2_NATIVE 1
#else
# define HAVE_SSE2(features) ((features) & X86_CPU_FEATURE_SSE2)
# define HAVE_SSE2_NATIVE 0
#endif
#if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
(defined(_MSC_VER) && defined(__AVX2__))
# define HAVE_PCLMULQDQ(features) 1
#else
# define HAVE_PCLMULQDQ(features) ((features) & X86_CPU_FEATURE_PCLMULQDQ)
#endif
#ifdef __AVX__
# define HAVE_AVX(features) 1
#else
# define HAVE_AVX(features) ((features) & X86_CPU_FEATURE_AVX)
#endif
#ifdef __AVX2__
# define HAVE_AVX2(features) 1
#else
# define HAVE_AVX2(features) ((features) & X86_CPU_FEATURE_AVX2)
#endif
#if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
# define HAVE_BMI2(features) 1
# define HAVE_BMI2_NATIVE 1
#else
# define HAVE_BMI2(features) ((features) & X86_CPU_FEATURE_BMI2)
# define HAVE_BMI2_NATIVE 0
#endif
#ifdef __AVX512BW__
# define HAVE_AVX512BW(features) 1
#else
# define HAVE_AVX512BW(features) ((features) & X86_CPU_FEATURE_AVX512BW)
#endif
#ifdef __AVX512VL__
# define HAVE_AVX512VL(features) 1
#else
# define HAVE_AVX512VL(features) ((features) & X86_CPU_FEATURE_AVX512VL)
#endif
#ifdef __VPCLMULQDQ__
# define HAVE_VPCLMULQDQ(features) 1
#else
# define HAVE_VPCLMULQDQ(features) ((features) & X86_CPU_FEATURE_VPCLMULQDQ)
#endif
#ifdef __AVX512VNNI__
# define HAVE_AVX512VNNI(features) 1
#else
# define HAVE_AVX512VNNI(features) ((features) & X86_CPU_FEATURE_AVX512VNNI)
#endif
#ifdef __AVXVNNI__
# define HAVE_AVXVNNI(features) 1
#else
# define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
#endif
#endif /* ARCH_X86_32 || ARCH_X86_64 */
#endif /* LIB_X86_CPU_FEATURES_H */
src/performance/libdeflate/lib/x86/crc32_impl.h
+159
View File
@@ -0,0 +1,159 @@
/*
* x86/crc32_impl.h - x86 implementations of the gzip CRC-32 algorithm
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef LIB_X86_CRC32_IMPL_H
#define LIB_X86_CRC32_IMPL_H
#include "cpu_features.h"
/*
* pshufb(x, shift_tab[len..len+15]) left shifts x by 16-len bytes.
* pshufb(x, shift_tab[len+16..len+31]) right shifts x by len bytes.
*/
static const u8 MAYBE_UNUSED shift_tab[48] = {
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
};
#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
/*
* PCLMULQDQ implementation. This targets PCLMULQDQ+SSE4.1, since in practice
* all CPUs that support PCLMULQDQ also support SSE4.1.
*/
# define crc32_x86_pclmulqdq crc32_x86_pclmulqdq
# define SUFFIX _pclmulqdq
# define ATTRIBUTES _target_attribute("pclmul,sse4.1")
# define VL 16
# define USE_AVX512 0
# include "crc32_pclmul_template.h"
/*
* PCLMULQDQ/AVX implementation. Same as above, but this is compiled with AVX
* enabled so that the compiler can generate VEX-coded instructions which can be
* slightly more efficient. It still uses 128-bit vectors.
*/
# define crc32_x86_pclmulqdq_avx crc32_x86_pclmulqdq_avx
# define SUFFIX _pclmulqdq_avx
# define ATTRIBUTES _target_attribute("pclmul,avx")
# define VL 16
# define USE_AVX512 0
# include "crc32_pclmul_template.h"
#endif
/*
* VPCLMULQDQ/AVX2 implementation. This is used on CPUs that have AVX2 and
* VPCLMULQDQ but don't have AVX-512, for example Intel Alder Lake.
*
* Currently this can't be enabled with MSVC because MSVC has a bug where it
* incorrectly assumes that VPCLMULQDQ implies AVX-512:
* https://developercommunity.visualstudio.com/t/Compiler-incorrectly-assumes-VAES-and-VP/10578785
*
* gcc 8.1 and 8.2 had a similar bug where they assumed that
* _mm256_clmulepi64_epi128() always needed AVX512. It's fixed in gcc 8.3.
*
* _mm256_zextsi128_si256() requires gcc 10.
*/
#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000)) && \
!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
# define crc32_x86_vpclmulqdq_avx2 crc32_x86_vpclmulqdq_avx2
# define SUFFIX _vpclmulqdq_avx2
# define ATTRIBUTES _target_attribute("vpclmulqdq,pclmul,avx2")
# define VL 32
# define USE_AVX512 0
# include "crc32_pclmul_template.h"
#endif
#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
/*
* VPCLMULQDQ/AVX512 implementation using 256-bit vectors. This is very similar
* to the VPCLMULQDQ/AVX2 implementation but takes advantage of the vpternlog
* instruction and more registers. This is used on certain older Intel CPUs,
* specifically Ice Lake and Tiger Lake, which support VPCLMULQDQ and AVX512 but
* downclock a bit too eagerly when ZMM registers are used.
*
* _mm256_zextsi128_si256() requires gcc 10.
*/
# define crc32_x86_vpclmulqdq_avx512_vl256 crc32_x86_vpclmulqdq_avx512_vl256
# define SUFFIX _vpclmulqdq_avx512_vl256
# define ATTRIBUTES _target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
# define VL 32
# define USE_AVX512 1
# include "crc32_pclmul_template.h"
/*
* VPCLMULQDQ/AVX512 implementation using 512-bit vectors. This is used on CPUs
* that have a good AVX-512 implementation including VPCLMULQDQ.
*
* _mm512_zextsi128_si512() requires gcc 10.
*/
# define crc32_x86_vpclmulqdq_avx512_vl512 crc32_x86_vpclmulqdq_avx512_vl512
# define SUFFIX _vpclmulqdq_avx512_vl512
# define ATTRIBUTES _target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
# define VL 64
# define USE_AVX512 1
# include "crc32_pclmul_template.h"
#endif
static inline crc32_func_t
arch_select_crc32_func(void)
{
const u32 features MAYBE_UNUSED = get_x86_cpu_features();
#ifdef crc32_x86_vpclmulqdq_avx512_vl512
if ((features & X86_CPU_FEATURE_ZMM) &&
HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
return crc32_x86_vpclmulqdq_avx512_vl512;
#endif
#ifdef crc32_x86_vpclmulqdq_avx512_vl256
if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
return crc32_x86_vpclmulqdq_avx512_vl256;
#endif
#ifdef crc32_x86_vpclmulqdq_avx2
if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
HAVE_AVX2(features))
return crc32_x86_vpclmulqdq_avx2;
#endif
#ifdef crc32_x86_pclmulqdq_avx
if (HAVE_PCLMULQDQ(features) && HAVE_AVX(features))
return crc32_x86_pclmulqdq_avx;
#endif
#ifdef crc32_x86_pclmulqdq
if (HAVE_PCLMULQDQ(features))
return crc32_x86_pclmulqdq;
#endif
return NULL;
}
#define arch_select_crc32_func arch_select_crc32_func
#endif /* LIB_X86_CRC32_IMPL_H */
src/performance/libdeflate/lib/x86/crc32_pclmul_template.h
+424
View File
@@ -0,0 +1,424 @@
/*
* x86/crc32_pclmul_template.h - gzip CRC-32 with PCLMULQDQ instructions
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
/*
* This file is a "template" for instantiating PCLMULQDQ-based crc32_x86
* functions. The "parameters" are:
*
* SUFFIX:
* Name suffix to append to all instantiated functions.
* ATTRIBUTES:
* Target function attributes to use. Must satisfy the dependencies of the
* other parameters as follows:
* VL=16 && USE_AVX512=0: at least pclmul,sse4.1
* VL=32 && USE_AVX512=0: at least vpclmulqdq,pclmul,avx2
* VL=32 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
* VL=64 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
* (Other combinations are not useful and have not been tested.)
* VL:
* Vector length in bytes. Must be 16, 32, or 64.
* USE_AVX512:
* If 1, take advantage of AVX-512 features such as masking and the
* vpternlog instruction. This doesn't enable the use of 512-bit vectors;
* the vector length is controlled by VL. If 0, assume that the CPU might
* not support AVX-512.
*
* The overall algorithm used is CRC folding with carryless multiplication
* instructions. Note that the x86 crc32 instruction cannot be used, as it is
* for a different polynomial, not the gzip one. For an explanation of CRC
* folding with carryless multiplication instructions, see
* scripts/gen-crc32-consts.py and the following blog posts and papers:
*
* "An alternative exposition of crc32_4k_pclmulqdq"
* https://www.corsix.org/content/alternative-exposition-crc32_4k_pclmulqdq
*
* "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
* https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
*
* The original pclmulqdq instruction does one 64x64 to 128-bit carryless
* multiplication. The VPCLMULQDQ feature added instructions that do two
* parallel 64x64 to 128-bit carryless multiplications in combination with AVX
* or AVX512VL, or four in combination with AVX512F.
*/
#if VL == 16
# define vec_t __m128i
# define fold_vec fold_vec128
# define VLOADU(p) _mm_loadu_si128((const void *)(p))
# define VXOR(a, b) _mm_xor_si128((a), (b))
# define M128I_TO_VEC(a) a
# define MULTS_8V _mm_set_epi64x(CRC32_X991_MODG, CRC32_X1055_MODG)
# define MULTS_4V _mm_set_epi64x(CRC32_X479_MODG, CRC32_X543_MODG)
# define MULTS_2V _mm_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG)
# define MULTS_1V _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG)
#elif VL == 32
# define vec_t __m256i
# define fold_vec fold_vec256
# define VLOADU(p) _mm256_loadu_si256((const void *)(p))
# define VXOR(a, b) _mm256_xor_si256((a), (b))
# define M128I_TO_VEC(a) _mm256_zextsi128_si256(a)
# define MULTS(a, b) _mm256_set_epi64x(a, b, a, b)
# define MULTS_8V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
# define MULTS_4V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
# define MULTS_2V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
# define MULTS_1V MULTS(CRC32_X223_MODG, CRC32_X287_MODG)
#elif VL == 64
# define vec_t __m512i
# define fold_vec fold_vec512
# define VLOADU(p) _mm512_loadu_si512((const void *)(p))
# define VXOR(a, b) _mm512_xor_si512((a), (b))
# define M128I_TO_VEC(a) _mm512_zextsi128_si512(a)
# define MULTS(a, b) _mm512_set_epi64(a, b, a, b, a, b, a, b)
# define MULTS_8V MULTS(CRC32_X4063_MODG, CRC32_X4127_MODG)
# define MULTS_4V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
# define MULTS_2V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
# define MULTS_1V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
#else
# error "unsupported vector length"
#endif
#undef fold_vec128
static forceinline ATTRIBUTES __m128i
ADD_SUFFIX(fold_vec128)(__m128i src, __m128i dst, __m128i /* __v2du */ mults)
{
dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x00));
dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x11));
return dst;
}
#define fold_vec128 ADD_SUFFIX(fold_vec128)
#if VL >= 32
#undef fold_vec256
static forceinline ATTRIBUTES __m256i
ADD_SUFFIX(fold_vec256)(__m256i src, __m256i dst, __m256i /* __v4du */ mults)
{
#if USE_AVX512
/* vpternlog with immediate 0x96 is a three-argument XOR. */
return _mm256_ternarylogic_epi32(
_mm256_clmulepi64_epi128(src, mults, 0x00),
_mm256_clmulepi64_epi128(src, mults, 0x11),
dst,
0x96);
#else
return _mm256_xor_si256(
_mm256_xor_si256(dst,
_mm256_clmulepi64_epi128(src, mults, 0x00)),
_mm256_clmulepi64_epi128(src, mults, 0x11));
#endif
}
#define fold_vec256 ADD_SUFFIX(fold_vec256)
#endif /* VL >= 32 */
#if VL >= 64
#undef fold_vec512
static forceinline ATTRIBUTES __m512i
ADD_SUFFIX(fold_vec512)(__m512i src, __m512i dst, __m512i /* __v8du */ mults)
{
/* vpternlog with immediate 0x96 is a three-argument XOR. */
return _mm512_ternarylogic_epi32(
_mm512_clmulepi64_epi128(src, mults, 0x00),
_mm512_clmulepi64_epi128(src, mults, 0x11),
dst,
0x96);
}
#define fold_vec512 ADD_SUFFIX(fold_vec512)
#endif /* VL >= 64 */
/*
* Given 'x' containing a 16-byte polynomial, and a pointer 'p' that points to
* the next '1 <= len <= 15' data bytes, rearrange the concatenation of 'x' and
* the data into vectors x0 and x1 that contain 'len' bytes and 16 bytes,
* respectively. Then fold x0 into x1 and return the result.
* Assumes that 'p + len - 16' is in-bounds.
*/
#undef fold_lessthan16bytes
static forceinline ATTRIBUTES __m128i
ADD_SUFFIX(fold_lessthan16bytes)(__m128i x, const u8 *p, size_t len,
__m128i /* __v2du */ mults_128b)
{
__m128i lshift = _mm_loadu_si128((const void *)&shift_tab[len]);
__m128i rshift = _mm_loadu_si128((const void *)&shift_tab[len + 16]);
__m128i x0, x1;
/* x0 = x left-shifted by '16 - len' bytes */
x0 = _mm_shuffle_epi8(x, lshift);
/*
* x1 = the last '16 - len' bytes from x (i.e. x right-shifted by 'len'
* bytes) followed by the remaining data.
*/
x1 = _mm_blendv_epi8(_mm_shuffle_epi8(x, rshift),
_mm_loadu_si128((const void *)(p + len - 16)),
/* msb 0/1 of each byte selects byte from arg1/2 */
rshift);
return fold_vec128(x0, x1, mults_128b);
}
#define fold_lessthan16bytes ADD_SUFFIX(fold_lessthan16bytes)
static ATTRIBUTES u32
ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
{
/*
* mults_{N}v are the vectors of multipliers for folding across N vec_t
* vectors, i.e. N*VL*8 bits. mults_128b are the two multipliers for
* folding across 128 bits. mults_128b differs from mults_1v when
* VL != 16. All multipliers are 64-bit, to match what pclmulqdq needs,
* but since this is for CRC-32 only their low 32 bits are nonzero.
* For more details, see scripts/gen-crc32-consts.py.
*/
const vec_t mults_8v = MULTS_8V;
const vec_t mults_4v = MULTS_4V;
const vec_t mults_2v = MULTS_2V;
const vec_t mults_1v = MULTS_1V;
const __m128i mults_128b = _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG);
const __m128i barrett_reduction_constants =
_mm_set_epi64x(CRC32_BARRETT_CONSTANT_2, CRC32_BARRETT_CONSTANT_1);
vec_t v0, v1, v2, v3, v4, v5, v6, v7;
__m128i x0 = _mm_cvtsi32_si128(crc);
__m128i x1;
if (len < 8*VL) {
if (len < VL) {
STATIC_ASSERT(VL == 16 || VL == 32 || VL == 64);
if (len < 16) {
#if USE_AVX512
if (len < 4)
return crc32_slice1(crc, p, len);
/*
* Handle 4 <= len <= 15 bytes by doing a masked
* load, XOR'ing the current CRC with the first
* 4 bytes, left-shifting by '16 - len' bytes to
* align the result to the end of x0 (so that it
* becomes the low-order coefficients of a
* 128-bit polynomial), and then doing the usual
* reduction from 128 bits to 32 bits.
*/
x0 = _mm_xor_si128(
x0, _mm_maskz_loadu_epi8((1 << len) - 1, p));
x0 = _mm_shuffle_epi8(
x0, _mm_loadu_si128((const void *)&shift_tab[len]));
goto reduce_x0;
#else
return crc32_slice1(crc, p, len);
#endif
}
/*
* Handle 16 <= len < VL bytes where VL is 32 or 64.
* Use 128-bit instructions so that these lengths aren't
* slower with VL > 16 than with VL=16.
*/
x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
if (len >= 32) {
x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 16)),
mults_128b);
if (len >= 48)
x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 32)),
mults_128b);
}
p += len & ~15;
goto less_than_16_remaining;
}
v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
if (len < 2*VL) {
p += VL;
goto less_than_vl_remaining;
}
v1 = VLOADU(p + 1*VL);
if (len < 4*VL) {
p += 2*VL;
goto less_than_2vl_remaining;
}
v2 = VLOADU(p + 2*VL);
v3 = VLOADU(p + 3*VL);
p += 4*VL;
} else {
/*
* If the length is large and the pointer is misaligned, align
* it. For smaller lengths, just take the misaligned load
* penalty. Note that on recent x86 CPUs, vmovdqu with an
* aligned address is just as fast as vmovdqa, so there's no
* need to use vmovdqa in the main loop.
*/
if (len > 65536 && ((uintptr_t)p & (VL-1))) {
size_t align = -(uintptr_t)p & (VL-1);
len -= align;
x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
p += 16;
if (align & 15) {
x0 = fold_lessthan16bytes(x0, p, align & 15,
mults_128b);
p += align & 15;
align &= ~15;
}
while (align) {
x0 = fold_vec128(x0, *(const __m128i *)p,
mults_128b);
p += 16;
align -= 16;
}
v0 = M128I_TO_VEC(x0);
# if VL == 32
v0 = _mm256_inserti128_si256(v0, *(const __m128i *)p, 1);
# elif VL == 64
v0 = _mm512_inserti32x4(v0, *(const __m128i *)p, 1);
v0 = _mm512_inserti64x4(v0, *(const __m256i *)(p + 16), 1);
# endif
p -= 16;
} else {
v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
}
v1 = VLOADU(p + 1*VL);
v2 = VLOADU(p + 2*VL);
v3 = VLOADU(p + 3*VL);
v4 = VLOADU(p + 4*VL);
v5 = VLOADU(p + 5*VL);
v6 = VLOADU(p + 6*VL);
v7 = VLOADU(p + 7*VL);
p += 8*VL;
/*
* This is the main loop, processing 8*VL bytes per iteration.
* 4*VL is usually enough and would result in smaller code, but
* Skylake and Cascade Lake need 8*VL to get full performance.
*/
while (len >= 16*VL) {
v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_8v);
v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_8v);
v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_8v);
v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_8v);
v4 = fold_vec(v4, VLOADU(p + 4*VL), mults_8v);
v5 = fold_vec(v5, VLOADU(p + 5*VL), mults_8v);
v6 = fold_vec(v6, VLOADU(p + 6*VL), mults_8v);
v7 = fold_vec(v7, VLOADU(p + 7*VL), mults_8v);
p += 8*VL;
len -= 8*VL;
}
/* Fewer than 8*VL bytes remain. */
v0 = fold_vec(v0, v4, mults_4v);
v1 = fold_vec(v1, v5, mults_4v);
v2 = fold_vec(v2, v6, mults_4v);
v3 = fold_vec(v3, v7, mults_4v);
if (len & (4*VL)) {
v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_4v);
v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_4v);
v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_4v);
v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_4v);
p += 4*VL;
}
}
/* Fewer than 4*VL bytes remain. */
v0 = fold_vec(v0, v2, mults_2v);
v1 = fold_vec(v1, v3, mults_2v);
if (len & (2*VL)) {
v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_2v);
v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_2v);
p += 2*VL;
}
less_than_2vl_remaining:
/* Fewer than 2*VL bytes remain. */
v0 = fold_vec(v0, v1, mults_1v);
if (len & VL) {
v0 = fold_vec(v0, VLOADU(p), mults_1v);
p += VL;
}
less_than_vl_remaining:
/*
* Fewer than VL bytes remain. Reduce v0 (length VL bytes) to x0
* (length 16 bytes) and fold in any 16-byte data segments that remain.
*/
#if VL == 16
x0 = v0;
#else
{
#if VL == 32
__m256i y0 = v0;
#else
const __m256i mults_256b =
_mm256_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG,
CRC32_X223_MODG, CRC32_X287_MODG);
__m256i y0 = fold_vec256(_mm512_extracti64x4_epi64(v0, 0),
_mm512_extracti64x4_epi64(v0, 1),
mults_256b);
if (len & 32) {
y0 = fold_vec256(y0, _mm256_loadu_si256((const void *)p),
mults_256b);
p += 32;
}
#endif
x0 = fold_vec128(_mm256_extracti128_si256(y0, 0),
_mm256_extracti128_si256(y0, 1), mults_128b);
}
if (len & 16) {
x0 = fold_vec128(x0, _mm_loadu_si128((const void *)p),
mults_128b);
p += 16;
}
#endif
less_than_16_remaining:
len &= 15;
/* Handle any remainder of 1 to 15 bytes. */
if (len)
x0 = fold_lessthan16bytes(x0, p, len, mults_128b);
#if USE_AVX512
reduce_x0:
#endif
/*
* Multiply the remaining 128-bit message polynomial 'x0' by x^32, then
* reduce it modulo the generator polynomial G. This gives the CRC.
*
* This implementation matches that used in crc-pclmul-template.S from
* https://lore.kernel.org/r/20250210174540.161705-4-ebiggers@kernel.org/
* with the parameters n=32 and LSB_CRC=1 (what the gzip CRC uses). See
* there for a detailed explanation of the math used here.
*/
x0 = _mm_xor_si128(_mm_clmulepi64_si128(x0, mults_128b, 0x10),
_mm_bsrli_si128(x0, 8));
x1 = _mm_clmulepi64_si128(x0, barrett_reduction_constants, 0x00);
x1 = _mm_clmulepi64_si128(x1, barrett_reduction_constants, 0x10);
x0 = _mm_xor_si128(x0, x1);
return _mm_extract_epi32(x0, 2);
}
#undef vec_t
#undef fold_vec
#undef VLOADU
#undef VXOR
#undef M128I_TO_VEC
#undef MULTS
#undef MULTS_8V
#undef MULTS_4V
#undef MULTS_2V
#undef MULTS_1V
#undef SUFFIX
#undef ATTRIBUTES
#undef VL
#undef USE_AVX512
src/performance/libdeflate/lib/x86/decompress_impl.h
+57
View File
@@ -0,0 +1,57 @@
#ifndef LIB_X86_DECOMPRESS_IMPL_H
#define LIB_X86_DECOMPRESS_IMPL_H
#include "cpu_features.h"
/*
* BMI2 optimized decompression function.
*
* With gcc and clang we just compile the whole function with
* __attribute__((target("bmi2"))), and the compiler uses bmi2 automatically.
*
* With MSVC, there is no target function attribute, but it's still possible to
* use bmi2 intrinsics explicitly. Currently we mostly don't, but there's a
* case in which we do (see below), so we at least take advantage of that.
* However, MSVC from VS2017 (toolset v141) apparently miscompiles the _bzhi_*()
* intrinsics. It seems to be fixed in VS2022. Hence, use MSVC_PREREQ(1930).
*/
#if defined(__GNUC__) || defined(__clang__) || MSVC_PREREQ(1930)
# define deflate_decompress_bmi2 deflate_decompress_bmi2
# define FUNCNAME deflate_decompress_bmi2
# define ATTRIBUTES _target_attribute("bmi2")
/*
* Even with __attribute__((target("bmi2"))), gcc doesn't reliably use the
* bzhi instruction for 'word & BITMASK(count)'. So use the bzhi intrinsic
* explicitly. EXTRACT_VARBITS() is equivalent to 'word & BITMASK(count)';
* EXTRACT_VARBITS8() is equivalent to 'word & BITMASK((u8)count)'.
* Nevertheless, their implementation using the bzhi intrinsic is identical,
* as the bzhi instruction truncates the count to 8 bits implicitly.
*/
# ifndef __clang__
# ifdef ARCH_X86_64
# define EXTRACT_VARBITS(word, count) _bzhi_u64((word), (count))
# define EXTRACT_VARBITS8(word, count) _bzhi_u64((word), (count))
# else
# define EXTRACT_VARBITS(word, count) _bzhi_u32((word), (count))
# define EXTRACT_VARBITS8(word, count) _bzhi_u32((word), (count))
# endif
# endif
# include "../decompress_template.h"
#endif
#if defined(deflate_decompress_bmi2) && HAVE_BMI2_NATIVE
#define DEFAULT_IMPL deflate_decompress_bmi2
#else
static inline decompress_func_t
arch_select_decompress_func(void)
{
#ifdef deflate_decompress_bmi2
if (HAVE_BMI2(get_x86_cpu_features()))
return deflate_decompress_bmi2;
#endif
return NULL;
}
#define arch_select_decompress_func arch_select_decompress_func
#endif
#endif /* LIB_X86_DECOMPRESS_IMPL_H */
src/performance/libdeflate/lib/x86/matchfinder_impl.h
+122
View File
@@ -0,0 +1,122 @@
/*
* x86/matchfinder_impl.h - x86 implementations of matchfinder functions
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef LIB_X86_MATCHFINDER_IMPL_H
#define LIB_X86_MATCHFINDER_IMPL_H
#include "cpu_features.h"
#ifdef __AVX2__
static forceinline void
matchfinder_init_avx2(mf_pos_t *data, size_t size)
{
__m256i *p = (__m256i *)data;
__m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
STATIC_ASSERT(sizeof(mf_pos_t) == 2);
do {
p[0] = v;
p[1] = v;
p[2] = v;
p[3] = v;
p += 4;
size -= 4 * sizeof(*p);
} while (size != 0);
}
#define matchfinder_init matchfinder_init_avx2
static forceinline void
matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
{
__m256i *p = (__m256i *)data;
__m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
STATIC_ASSERT(sizeof(mf_pos_t) == 2);
do {
/* PADDSW: Add Packed Signed Integers With Signed Saturation */
p[0] = _mm256_adds_epi16(p[0], v);
p[1] = _mm256_adds_epi16(p[1], v);
p[2] = _mm256_adds_epi16(p[2], v);
p[3] = _mm256_adds_epi16(p[3], v);
p += 4;
size -= 4 * sizeof(*p);
} while (size != 0);
}
#define matchfinder_rebase matchfinder_rebase_avx2
#elif HAVE_SSE2_NATIVE
static forceinline void
matchfinder_init_sse2(mf_pos_t *data, size_t size)
{
__m128i *p = (__m128i *)data;
__m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL);
STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
STATIC_ASSERT(sizeof(mf_pos_t) == 2);
do {
p[0] = v;
p[1] = v;
p[2] = v;
p[3] = v;
p += 4;
size -= 4 * sizeof(*p);
} while (size != 0);
}
#define matchfinder_init matchfinder_init_sse2
static forceinline void
matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
{
__m128i *p = (__m128i *)data;
__m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
STATIC_ASSERT(sizeof(mf_pos_t) == 2);
do {
/* PADDSW: Add Packed Signed Integers With Signed Saturation */
p[0] = _mm_adds_epi16(p[0], v);
p[1] = _mm_adds_epi16(p[1], v);
p[2] = _mm_adds_epi16(p[2], v);
p[3] = _mm_adds_epi16(p[3], v);
p += 4;
size -= 4 * sizeof(*p);
} while (size != 0);
}
#define matchfinder_rebase matchfinder_rebase_sse2
#endif /* HAVE_SSE2_NATIVE */
#endif /* LIB_X86_MATCHFINDER_IMPL_H */
src/performance/libdeflate/lib/zlib_constants.h
+21
View File
@@ -0,0 +1,21 @@
/*
* zlib_constants.h - constants for the zlib wrapper format
*/
#ifndef LIB_ZLIB_CONSTANTS_H
#define LIB_ZLIB_CONSTANTS_H
#define ZLIB_MIN_HEADER_SIZE 2
#define ZLIB_FOOTER_SIZE 4
#define ZLIB_MIN_OVERHEAD (ZLIB_MIN_HEADER_SIZE + ZLIB_FOOTER_SIZE)
#define ZLIB_CM_DEFLATE 8
#define ZLIB_CINFO_32K_WINDOW 7
#define ZLIB_FASTEST_COMPRESSION 0
#define ZLIB_FAST_COMPRESSION 1
#define ZLIB_DEFAULT_COMPRESSION 2
#define ZLIB_SLOWEST_COMPRESSION 3
#endif /* LIB_ZLIB_CONSTANTS_H */
src/performance/libdeflate/lib/zlib_decompress.c
+104
View File
@@ -0,0 +1,104 @@
/*
* zlib_decompress.c - decompress with a zlib wrapper
*
* Copyright 2016 Eric Biggers
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "lib_common.h"
#include "zlib_constants.h"
LIBDEFLATEAPI enum libdeflate_result
libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail,
size_t *actual_in_nbytes_ret,
size_t *actual_out_nbytes_ret)
{
const u8 *in_next = in;
const u8 * const in_end = in_next + in_nbytes;
u16 hdr;
size_t actual_in_nbytes;
size_t actual_out_nbytes;
enum libdeflate_result result;
if (in_nbytes < ZLIB_MIN_OVERHEAD)
return LIBDEFLATE_BAD_DATA;
/* 2 byte header: CMF and FLG */
hdr = get_unaligned_be16(in_next);
in_next += 2;
/* FCHECK */
if ((hdr % 31) != 0)
return LIBDEFLATE_BAD_DATA;
/* CM */
if (((hdr >> 8) & 0xF) != ZLIB_CM_DEFLATE)
return LIBDEFLATE_BAD_DATA;
/* CINFO */
if ((hdr >> 12) > ZLIB_CINFO_32K_WINDOW)
return LIBDEFLATE_BAD_DATA;
/* FDICT */
if ((hdr >> 5) & 1)
return LIBDEFLATE_BAD_DATA;
/* Compressed data */
result = libdeflate_deflate_decompress_ex(d, in_next,
in_end - ZLIB_FOOTER_SIZE - in_next,
out, out_nbytes_avail,
&actual_in_nbytes, actual_out_nbytes_ret);
if (result != LIBDEFLATE_SUCCESS)
return result;
if (actual_out_nbytes_ret)
actual_out_nbytes = *actual_out_nbytes_ret;
else
actual_out_nbytes = out_nbytes_avail;
in_next += actual_in_nbytes;
/* ADLER32 */
if (libdeflate_adler32(1, out, actual_out_nbytes) !=
get_unaligned_be32(in_next))
return LIBDEFLATE_BAD_DATA;
in_next += 4;
if (actual_in_nbytes_ret)
*actual_in_nbytes_ret = in_next - (u8 *)in;
return LIBDEFLATE_SUCCESS;
}
LIBDEFLATEAPI enum libdeflate_result
libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail,
size_t *actual_out_nbytes_ret)
{
return libdeflate_zlib_decompress_ex(d, in, in_nbytes,
out, out_nbytes_avail,
NULL, actual_out_nbytes_ret);
}
src/performance/libdeflate/libdeflate.h
+411
View File
@@ -0,0 +1,411 @@
/*
* libdeflate.h - public header for libdeflate
*/
#ifndef LIBDEFLATE_H
#define LIBDEFLATE_H
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#define LIBDEFLATE_VERSION_MAJOR 1
#define LIBDEFLATE_VERSION_MINOR 25
#define LIBDEFLATE_VERSION_STRING "1.25"
/*
* Users of libdeflate.dll on Windows can define LIBDEFLATE_DLL to cause
* __declspec(dllimport) to be used. This should be done when it's easy to do.
* Otherwise it's fine to skip it, since it is a very minor performance
* optimization that is irrelevant for most use cases of libdeflate.
*/
#ifndef LIBDEFLATEAPI
# if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
# define LIBDEFLATEAPI __declspec(dllimport)
# else
# define LIBDEFLATEAPI
# endif
#endif
/* ========================================================================== */
/* Compression */
/* ========================================================================== */
struct libdeflate_compressor;
struct libdeflate_options;
/*
* libdeflate_alloc_compressor() allocates a new compressor that supports
* DEFLATE, zlib, and gzip compression. 'compression_level' is the compression
* level on a zlib-like scale but with a higher maximum value (1 = fastest, 6 =
* medium/default, 9 = slow, 12 = slowest). Level 0 is also supported and means
* "no compression", specifically "create a valid stream, but only emit
* uncompressed blocks" (this will expand the data slightly).
*
* The return value is a pointer to the new compressor, or NULL if out of memory
* or if the compression level is invalid (i.e. outside the range [0, 12]).
*
* Note: for compression, the sliding window size is defined at compilation time
* to 32768, the largest size permissible in the DEFLATE format. It cannot be
* changed at runtime.
*
* A single compressor is not safe to use by multiple threads concurrently.
* However, different threads may use different compressors concurrently.
*/
LIBDEFLATEAPI struct libdeflate_compressor *
libdeflate_alloc_compressor(int compression_level);
/*
* Like libdeflate_alloc_compressor(), but adds the 'options' argument.
*/
LIBDEFLATEAPI struct libdeflate_compressor *
libdeflate_alloc_compressor_ex(int compression_level,
const struct libdeflate_options *options);
/*
* libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of
* data. It attempts to compress 'in_nbytes' bytes of data located at 'in' and
* write the result to 'out', which has space for 'out_nbytes_avail' bytes. The
* return value is the compressed size in bytes, or 0 if the data could not be
* compressed to 'out_nbytes_avail' bytes or fewer.
*
* If compression is successful, then the output data is guaranteed to be a
* valid DEFLATE stream that decompresses to the input data. No other
* guarantees are made about the output data. Notably, different versions of
* libdeflate can produce different compressed data for the same uncompressed
* data, even at the same compression level. Do ***NOT*** do things like
* writing tests that compare compressed data to a golden output, as this can
* break when libdeflate is updated. (This property isn't specific to
* libdeflate; the same is true for zlib and other compression libraries too.)
*/
LIBDEFLATEAPI size_t
libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail);
/*
* libdeflate_deflate_compress_bound() returns a worst-case upper bound on the
* number of bytes of compressed data that may be produced by compressing any
* buffer of length less than or equal to 'in_nbytes' using
* libdeflate_deflate_compress() with the specified compressor. This bound will
* necessarily be a number greater than or equal to 'in_nbytes'. It may be an
* overestimate of the true upper bound. The return value is guaranteed to be
* the same for all invocations with the same compressor and same 'in_nbytes'.
*
* As a special case, 'compressor' may be NULL. This causes the bound to be
* taken across *any* libdeflate_compressor that could ever be allocated with
* this build of the library, with any options.
*
* Note that this function is not necessary in many applications. With
* block-based compression, it is usually preferable to separately store the
* uncompressed size of each block and to store any blocks that did not compress
* to less than their original size uncompressed. In that scenario, there is no
* need to know the worst-case compressed size, since the maximum number of
* bytes of compressed data that may be used would always be one less than the
* input length. You can just pass a buffer of that size to
* libdeflate_deflate_compress() and store the data uncompressed if
* libdeflate_deflate_compress() returns 0, indicating that the compressed data
* did not fit into the provided output buffer.
*/
LIBDEFLATEAPI size_t
libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
size_t in_nbytes);
/*
* Like libdeflate_deflate_compress(), but uses the zlib wrapper format instead
* of raw DEFLATE.
*/
LIBDEFLATEAPI size_t
libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail);
/*
* Like libdeflate_deflate_compress_bound(), but assumes the data will be
* compressed with libdeflate_zlib_compress() rather than with
* libdeflate_deflate_compress().
*/
LIBDEFLATEAPI size_t
libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
size_t in_nbytes);
/*
* Like libdeflate_deflate_compress(), but uses the gzip wrapper format instead
* of raw DEFLATE.
*/
LIBDEFLATEAPI size_t
libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail);
/*
* Like libdeflate_deflate_compress_bound(), but assumes the data will be
* compressed with libdeflate_gzip_compress() rather than with
* libdeflate_deflate_compress().
*/
LIBDEFLATEAPI size_t
libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
size_t in_nbytes);
/*
* libdeflate_free_compressor() frees a compressor that was allocated with
* libdeflate_alloc_compressor(). If a NULL pointer is passed in, no action is
* taken.
*/
LIBDEFLATEAPI void
libdeflate_free_compressor(struct libdeflate_compressor *compressor);
/* ========================================================================== */
/* Decompression */
/* ========================================================================== */
struct libdeflate_decompressor;
struct libdeflate_options;
/*
* libdeflate_alloc_decompressor() allocates a new decompressor that can be used
* for DEFLATE, zlib, and gzip decompression. The return value is a pointer to
* the new decompressor, or NULL if out of memory.
*
* This function takes no parameters, and the returned decompressor is valid for
* decompressing data that was compressed at any compression level and with any
* sliding window size.
*
* A single decompressor is not safe to use by multiple threads concurrently.
* However, different threads may use different decompressors concurrently.
*/
LIBDEFLATEAPI struct libdeflate_decompressor *
libdeflate_alloc_decompressor(void);
/*
* Like libdeflate_alloc_decompressor(), but adds the 'options' argument.
*/
LIBDEFLATEAPI struct libdeflate_decompressor *
libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
/*
* Result of a call to libdeflate_deflate_decompress(),
* libdeflate_zlib_decompress(), or libdeflate_gzip_decompress().
*/
enum libdeflate_result {
/* Decompression was successful. */
LIBDEFLATE_SUCCESS = 0,
/* Decompression failed because the compressed data was invalid,
* corrupt, or otherwise unsupported. */
LIBDEFLATE_BAD_DATA = 1,
/* A NULL 'actual_out_nbytes_ret' was provided, but the data would have
* decompressed to fewer than 'out_nbytes_avail' bytes. */
LIBDEFLATE_SHORT_OUTPUT = 2,
/* The data would have decompressed to more than 'out_nbytes_avail'
* bytes. */
LIBDEFLATE_INSUFFICIENT_SPACE = 3,
};
/*
* libdeflate_deflate_decompress() decompresses a DEFLATE stream from the buffer
* 'in' with compressed size up to 'in_nbytes' bytes. The uncompressed data is
* written to 'out', a buffer with size 'out_nbytes_avail' bytes. If
* decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned. Otherwise,
* a nonzero result code such as LIBDEFLATE_BAD_DATA is returned, and the
* contents of the output buffer are undefined.
*
* Decompression stops at the end of the DEFLATE stream (as indicated by the
* BFINAL flag), even if it is actually shorter than 'in_nbytes' bytes.
*
* libdeflate_deflate_decompress() can be used in cases where the actual
* uncompressed size is known (recommended) or unknown (not recommended):
*
* - If the actual uncompressed size is known, then pass the actual
* uncompressed size as 'out_nbytes_avail' and pass NULL for
* 'actual_out_nbytes_ret'. This makes libdeflate_deflate_decompress() fail
* with LIBDEFLATE_SHORT_OUTPUT if the data decompressed to fewer than the
* specified number of bytes.
*
* - If the actual uncompressed size is unknown, then provide a non-NULL
* 'actual_out_nbytes_ret' and provide a buffer with some size
* 'out_nbytes_avail' that you think is large enough to hold all the
* uncompressed data. In this case, if the data decompresses to less than
* or equal to 'out_nbytes_avail' bytes, then
* libdeflate_deflate_decompress() will write the actual uncompressed size
* to *actual_out_nbytes_ret and return 0 (LIBDEFLATE_SUCCESS). Otherwise,
* it will return LIBDEFLATE_INSUFFICIENT_SPACE if the provided buffer was
* not large enough but no other problems were encountered, or another
* nonzero result code if decompression failed for another reason.
*/
LIBDEFLATEAPI enum libdeflate_result
libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail,
size_t *actual_out_nbytes_ret);
/*
* Like libdeflate_deflate_decompress(), but adds the 'actual_in_nbytes_ret'
* argument. If decompression succeeds and 'actual_in_nbytes_ret' is not NULL,
* then the actual compressed size of the DEFLATE stream (aligned to the next
* byte boundary) is written to *actual_in_nbytes_ret.
*/
LIBDEFLATEAPI enum libdeflate_result
libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail,
size_t *actual_in_nbytes_ret,
size_t *actual_out_nbytes_ret);
/*
* Like libdeflate_deflate_decompress(), but assumes the zlib wrapper format
* instead of raw DEFLATE.
*
* Decompression will stop at the end of the zlib stream, even if it is shorter
* than 'in_nbytes'. If you need to know exactly where the zlib stream ended,
* use libdeflate_zlib_decompress_ex().
*/
LIBDEFLATEAPI enum libdeflate_result
libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail,
size_t *actual_out_nbytes_ret);
/*
* Like libdeflate_zlib_decompress(), but adds the 'actual_in_nbytes_ret'
* argument. If 'actual_in_nbytes_ret' is not NULL and the decompression
* succeeds (indicating that the first zlib-compressed stream in the input
* buffer was decompressed), then the actual number of input bytes consumed is
* written to *actual_in_nbytes_ret.
*/
LIBDEFLATEAPI enum libdeflate_result
libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail,
size_t *actual_in_nbytes_ret,
size_t *actual_out_nbytes_ret);
/*
* Like libdeflate_deflate_decompress(), but assumes the gzip wrapper format
* instead of raw DEFLATE.
*
* If multiple gzip-compressed members are concatenated, then only the first
* will be decompressed. Use libdeflate_gzip_decompress_ex() if you need
* multi-member support.
*/
LIBDEFLATEAPI enum libdeflate_result
libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail,
size_t *actual_out_nbytes_ret);
/*
* Like libdeflate_gzip_decompress(), but adds the 'actual_in_nbytes_ret'
* argument. If 'actual_in_nbytes_ret' is not NULL and the decompression
* succeeds (indicating that the first gzip-compressed member in the input
* buffer was decompressed), then the actual number of input bytes consumed is
* written to *actual_in_nbytes_ret.
*/
LIBDEFLATEAPI enum libdeflate_result
libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
const void *in, size_t in_nbytes,
void *out, size_t out_nbytes_avail,
size_t *actual_in_nbytes_ret,
size_t *actual_out_nbytes_ret);
/*
* libdeflate_free_decompressor() frees a decompressor that was allocated with
* libdeflate_alloc_decompressor(). If a NULL pointer is passed in, no action
* is taken.
*/
LIBDEFLATEAPI void
libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
/* ========================================================================== */
/* Checksums */
/* ========================================================================== */
/*
* libdeflate_adler32() updates a running Adler-32 checksum with 'len' bytes of
* data and returns the updated checksum. When starting a new checksum, the
* required initial value for 'adler' is 1. This value is also returned when
* 'buffer' is specified as NULL.
*/
LIBDEFLATEAPI uint32_t
libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
/*
* libdeflate_crc32() updates a running CRC-32 checksum with 'len' bytes of data
* and returns the updated checksum. When starting a new checksum, the required
* initial value for 'crc' is 0. This value is also returned when 'buffer' is
* specified as NULL.
*/
LIBDEFLATEAPI uint32_t
libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
/* ========================================================================== */
/* Custom memory allocator */
/* ========================================================================== */
/*
* Install a custom memory allocator which libdeflate will use for all memory
* allocations by default. 'malloc_func' is a function that must behave like
* malloc(), and 'free_func' is a function that must behave like free().
*
* The per-(de)compressor custom memory allocator that can be specified in
* 'struct libdeflate_options' takes priority over this.
*
* This doesn't affect the free() function that will be used to free
* (de)compressors that were already in existence when this is called.
*/
LIBDEFLATEAPI void
libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
void (*free_func)(void *));
/*
* Advanced options. This is the options structure that
* libdeflate_alloc_compressor_ex() and libdeflate_alloc_decompressor_ex()
* require. Most users won't need this and should just use the non-"_ex"
* functions instead. If you do need this, it should be initialized like this:
*
* struct libdeflate_options options;
*
* memset(&options, 0, sizeof(options));
* options.sizeof_options = sizeof(options);
* // Then set the fields that you need to override the defaults for.
*/
struct libdeflate_options {
/*
* This field must be set to the struct size. This field exists for
* extensibility, so that fields can be appended to this struct in
* future versions of libdeflate while still supporting old binaries.
*/
size_t sizeof_options;
/*
* An optional custom memory allocator to use for this (de)compressor.
* 'malloc_func' must be a function that behaves like malloc(), and
* 'free_func' must be a function that behaves like free().
*
* This is useful in cases where a process might have multiple users of
* libdeflate who want to use different memory allocators. For example,
* a library might want to use libdeflate with a custom memory allocator
* without interfering with user code that might use libdeflate too.
*
* This takes priority over the "global" memory allocator (which by
* default is malloc() and free(), but can be changed by
* libdeflate_set_memory_allocator()). Moreover, libdeflate will never
* call the "global" memory allocator if a per-(de)compressor custom
* allocator is always given.
*/
void *(*malloc_func)(size_t);
void (*free_func)(void *);
};
#ifdef __cplusplus
}
#endif
#endif /* LIBDEFLATE_H */