From f0ce8884c0588a89dff01ba8a9e989f6284ba495 Mon Sep 17 00:00:00 2001 From: John Naylor Date: Tue, 2 Mar 2021 12:24:49 -0400 Subject: [PATCH v1 2/4] Use platform-specific implementations of pg_popcount. Since this function takes a buffer that's likely much longer than the word size, we can better amortize the cost of an indirect function call. For the asm version, use the existing pg_popcount{32,64}_asm functions, but inline them. --- src/include/port/pg_bitutils.h | 2 +- src/port/pg_bitutils.c | 87 ++++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h index 2c40784830..708b9a6a67 100644 --- a/src/include/port/pg_bitutils.h +++ b/src/include/port/pg_bitutils.h @@ -212,7 +212,7 @@ extern int pg_popcount32 (uint32 word); extern int pg_popcount64 (uint64 word); /* Count the number of one-bits in a byte array */ -extern uint64 pg_popcount(const char *buf, int bytes); +extern uint64 (*pg_popcount) (const char *buf, int bytes); /* * Rotate the bits of "word" to the right by n bits. diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c index 5dab793e49..9be8b78cff 100644 --- a/src/port/pg_bitutils.c +++ b/src/port/pg_bitutils.c @@ -141,7 +141,7 @@ pg_popcount_available(void) * pg_popcount32_asm * Return the number of 1 bits set in word */ -static int +static inline int pg_popcount32_asm(uint32 word) { uint32 res; @@ -154,7 +154,7 @@ __asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc"); * pg_popcount64_asm * Return the number of 1 bits set in word */ -static int +static inline int pg_popcount64_asm(uint64 word) { uint64 res; @@ -165,6 +165,18 @@ __asm__ __volatile__(" popcntq %1,%0\n":"=q"(res):"rm"(word):"cc"); #endif /* USE_POPCNT_ASM */ +static uint64 pg_popcount_slow(const char *buf, int bytes); + +#ifdef USE_POPCNT_ASM +static bool pg_popcount_available(void); +static uint64 pg_popcount_choose(const char *buf, int bytes); +static uint64 pg_popcount_asm(const char *buf, int bytes); + +uint64 (*pg_popcount) (const char *buf, int bytes) = pg_popcount_choose; +#else +uint64 (*pg_popcount) (const char *buf, int bytes) = pg_popcount_slow; +#endif /* USE_POPCNT_ASM */ + /* * pg_popcount32 @@ -216,13 +228,30 @@ pg_popcount64(uint64 word) #endif /* HAVE__BUILTIN_POPCOUNT */ } +/* + * This function gets called on the first call to pg_popcount. + * It detects whether we can use the asm implementation, and replace + * the function pointer so that subsequent calls are routed directly to + * the chosen implementation. + */ +static uint64 +pg_popcount_choose(const char *buf, int bytes) +{ + if (pg_popcount_available()) + pg_popcount = pg_popcount_asm; + else + pg_popcount = pg_popcount_slow; + + return pg_popcount(buf, bytes); +} + /* - * pg_popcount + * pg_popcount_slow * Returns the number of 1-bits in buf */ uint64 -pg_popcount(const char *buf, int bytes) +pg_popcount_slow(const char *buf, int bytes) { uint64 popcnt = 0; @@ -262,3 +291,53 @@ pg_popcount(const char *buf, int bytes) return popcnt; } + +#ifdef USE_POPCNT_ASM + +/* + * pg_popcount_asm + * Returns the number of 1-bits in buf using POPCNT + */ +uint64 +pg_popcount_asm(const char *buf, int bytes) +{ + uint64 popcnt = 0; + +#if SIZEOF_VOID_P >= 8 + /* Process in 64-bit chunks if the buffer is aligned. */ + if (buf == (const char *) TYPEALIGN(8, buf)) + { + const uint64 *words = (const uint64 *) buf; + + while (bytes >= 8) + { + popcnt += pg_popcount64_asm(*words++); + bytes -= 8; + } + + buf = (const char *) words; + } +#else + /* Process in 32-bit chunks if the buffer is aligned. */ + if (buf == (const char *) TYPEALIGN(4, buf)) + { + const uint32 *words = (const uint32 *) buf; + + while (bytes >= 4) + { + popcnt += pg_popcount32_asm(*words++); + bytes -= 4; + } + + buf = (const char *) words; + } +#endif + + /* Process any remaining bytes */ + while (bytes--) + popcnt += pg_number_of_ones[(unsigned char) *buf++]; + + return popcnt; +} + +#endif /* USE_POPCNT_ASM */ -- 2.22.0