murmur3哈希算法

murmur3非加密哈希算法

murmur3非加密哈希算法导图

据算法作者Austin Appleby描述，有c1, c2, n 三个常量用大量测试数据调测出来的，可以对数值进行微调。

murmur3哈希算法实现

//-----------------------------------------------------------------------------
// MurmurHash3 was written by Austin Appleby, and is placed in the public
// domain. The author hereby disclaims copyright to this source code.// Note - The x86 and x64 versions do _not_ produce the same results, as the
// algorithms are optimized for their respective platforms. You can still
// compile and run any of them on any platform, but your performance with the
// non-native version will be less than optimal.#include "MurmurHash3.h"//-----------------------------------------------------------------------------
// Platform-specific functions and macros// Microsoft Visual Studio#if defined(_MSC_VER)#define FORCE_INLINE    __forceinline#include <stdlib.h>#define ROTL32(x,y)    _rotl(x,y)
#define ROTL64(x,y)    _rotl64(x,y)#define BIG_CONSTANT(x) (x)// Other compilers#else    // defined(_MSC_VER)#define    FORCE_INLINE inline __attribute__((always_inline))inline uint32_t rotl32 ( uint32_t x, int8_t r )
{return (x << r) | (x >> (32 - r));
}inline uint64_t rotl64 ( uint64_t x, int8_t r )
{return (x << r) | (x >> (64 - r));
}#define    ROTL32(x,y)    rotl32(x,y)
#define ROTL64(x,y)    rotl64(x,y)#define BIG_CONSTANT(x) (x##LLU)#endif // !defined(_MSC_VER)//-----------------------------------------------------------------------------
// Block read - if your platform needs to do endian-swapping or can only
// handle aligned reads, do the conversion hereFORCE_INLINE uint32_t getblock32 ( const uint32_t * p, int i )
{return p[i];
}FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, int i )
{return p[i];
}//-----------------------------------------------------------------------------
// Finalization mix - force all bits of a hash block to avalancheFORCE_INLINE uint32_t fmix32 ( uint32_t h )
{h ^= h >> 16;h *= 0x85ebca6b;h ^= h >> 13;h *= 0xc2b2ae35;h ^= h >> 16;return h;
}//----------FORCE_INLINE uint64_t fmix64 ( uint64_t k )
{k ^= k >> 33;k *= BIG_CONSTANT(0xff51afd7ed558ccd);k ^= k >> 33;k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53);k ^= k >> 33;return k;
}//-----------------------------------------------------------------------------void MurmurHash3_x86_32 ( const void * key, int len,uint32_t seed, void * out )
{const uint8_t * data = (const uint8_t*)key;const int nblocks = len / 4;uint32_t h1 = seed;const uint32_t c1 = 0xcc9e2d51;const uint32_t c2 = 0x1b873593;//----------// bodyconst uint32_t * blocks = (const uint32_t *)(data + nblocks*4);for(int i = -nblocks; i; i++){uint32_t k1 = getblock32(blocks,i);k1 *= c1;k1 = ROTL32(k1,15);k1 *= c2;h1 ^= k1;h1 = ROTL32(h1,13); h1 = h1*5+0xe6546b64;}//----------// tailconst uint8_t * tail = (const uint8_t*)(data + nblocks*4);uint32_t k1 = 0;switch(len & 3){case 3: k1 ^= tail[2] << 16;case 2: k1 ^= tail[1] << 8;case 1: k1 ^= tail[0];k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1;};//----------// finalizationh1 ^= len;h1 = fmix32(h1);*(uint32_t*)out = h1;
} //-----------------------------------------------------------------------------void MurmurHash3_x86_128 ( const void * key, const int len,uint32_t seed, void * out )
{const uint8_t * data = (const uint8_t*)key;const int nblocks = len / 16;uint32_t h1 = seed;uint32_t h2 = seed;uint32_t h3 = seed;uint32_t h4 = seed;const uint32_t c1 = 0x239b961b; const uint32_t c2 = 0xab0e9789;const uint32_t c3 = 0x38b34ae5; const uint32_t c4 = 0xa1e38b93;//----------// bodyconst uint32_t * blocks = (const uint32_t *)(data + nblocks*16);for(int i = -nblocks; i; i++){uint32_t k1 = getblock32(blocks,i*4+0);uint32_t k2 = getblock32(blocks,i*4+1);uint32_t k3 = getblock32(blocks,i*4+2);uint32_t k4 = getblock32(blocks,i*4+3);k1 *= c1; k1  = ROTL32(k1,15); k1 *= c2; h1 ^= k1;h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b;k2 *= c2; k2  = ROTL32(k2,16); k2 *= c3; h2 ^= k2;h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747;k3 *= c3; k3  = ROTL32(k3,17); k3 *= c4; h3 ^= k3;h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35;k4 *= c4; k4  = ROTL32(k4,18); k4 *= c1; h4 ^= k4;h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17;}//----------// tailconst uint8_t * tail = (const uint8_t*)(data + nblocks*16);uint32_t k1 = 0;uint32_t k2 = 0;uint32_t k3 = 0;uint32_t k4 = 0;switch(len & 15){case 15: k4 ^= tail[14] << 16;case 14: k4 ^= tail[13] << 8;case 13: k4 ^= tail[12] << 0;k4 *= c4; k4  = ROTL32(k4,18); k4 *= c1; h4 ^= k4;case 12: k3 ^= tail[11] << 24;case 11: k3 ^= tail[10] << 16;case 10: k3 ^= tail[ 9] << 8;case  9: k3 ^= tail[ 8] << 0;k3 *= c3; k3  = ROTL32(k3,17); k3 *= c4; h3 ^= k3;case  8: k2 ^= tail[ 7] << 24;case  7: k2 ^= tail[ 6] << 16;case  6: k2 ^= tail[ 5] << 8;case  5: k2 ^= tail[ 4] << 0;k2 *= c2; k2  = ROTL32(k2,16); k2 *= c3; h2 ^= k2;case  4: k1 ^= tail[ 3] << 24;case  3: k1 ^= tail[ 2] << 16;case  2: k1 ^= tail[ 1] << 8;case  1: k1 ^= tail[ 0] << 0;k1 *= c1; k1  = ROTL32(k1,15); k1 *= c2; h1 ^= k1;};//----------// finalizationh1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;h1 += h2; h1 += h3; h1 += h4;h2 += h1; h3 += h1; h4 += h1;h1 = fmix32(h1);h2 = fmix32(h2);h3 = fmix32(h3);h4 = fmix32(h4);h1 += h2; h1 += h3; h1 += h4;h2 += h1; h3 += h1; h4 += h1;((uint32_t*)out)[0] = h1;((uint32_t*)out)[1] = h2;((uint32_t*)out)[2] = h3;((uint32_t*)out)[3] = h4;
}//-----------------------------------------------------------------------------void MurmurHash3_x64_128 ( const void * key, const int len,const uint32_t seed, void * out )
{const uint8_t * data = (const uint8_t*)key;const int nblocks = len / 16;uint64_t h1 = seed;uint64_t h2 = seed;const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5);const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f);//----------// bodyconst uint64_t * blocks = (const uint64_t *)(data);for(int i = 0; i < nblocks; i++){uint64_t k1 = getblock64(blocks,i*2+0);uint64_t k2 = getblock64(blocks,i*2+1);k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; h1 ^= k1;h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729;k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; h2 ^= k2;h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5;}//----------// tailconst uint8_t * tail = (const uint8_t*)(data + nblocks*16);uint64_t k1 = 0;uint64_t k2 = 0;switch(len & 15){case 15: k2 ^= ((uint64_t)tail[14]) << 48;case 14: k2 ^= ((uint64_t)tail[13]) << 40;case 13: k2 ^= ((uint64_t)tail[12]) << 32;case 12: k2 ^= ((uint64_t)tail[11]) << 24;case 11: k2 ^= ((uint64_t)tail[10]) << 16;case 10: k2 ^= ((uint64_t)tail[ 9]) << 8;case  9: k2 ^= ((uint64_t)tail[ 8]) << 0;k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; h2 ^= k2;case  8: k1 ^= ((uint64_t)tail[ 7]) << 56;case  7: k1 ^= ((uint64_t)tail[ 6]) << 48;case  6: k1 ^= ((uint64_t)tail[ 5]) << 40;case  5: k1 ^= ((uint64_t)tail[ 4]) << 32;case  4: k1 ^= ((uint64_t)tail[ 3]) << 24;case  3: k1 ^= ((uint64_t)tail[ 2]) << 16;case  2: k1 ^= ((uint64_t)tail[ 1]) << 8;case  1: k1 ^= ((uint64_t)tail[ 0]) << 0;k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; h1 ^= k1;};//----------// finalizationh1 ^= len; h2 ^= len;h1 += h2;h2 += h1;h1 = fmix64(h1);h2 = fmix64(h2);h1 += h2;h2 += h1;((uint64_t*)out)[0] = h1;((uint64_t*)out)[1] = h2;
}//-----------------------------------------------------------------------------

官方测试用例和个人理解

Murmur3 可用性测试验证示例

template < typename hashtype >
void test ( hashfunc<hashtype> hash, HashInfo * info )
{const int hashbits = sizeof(hashtype) * 8;printf("-------------------------------------------------------------------------------\n");printf("--- Testing %s (%s)\n\n",info->name,info->desc);//-----------------------------------------------------------------------------// Sanity tests 可用性测试if(g_testSanity || g_testAll){printf("[[[ Sanity Tests ]]]\n\n");VerificationTest(hash,hashbits,info->verification,true);SanityTest(hash,hashbits);AppendedZeroesTest(hash,hashbits);printf("\n");}

验证测试代码：

对于数组作为键值的用法来说，影响hash值的因素可以通过VerificationTest函数窥探出：

1、数组的大小(如{1,2}和{1,2,3})
2、数组的内容填充（如{1,2}和{2,3}）
3、数组的排列组合顺序（如{2,1}和{1,2}）

实现逻辑：

如数组{0,1,2,...,n}
先对{0}取murmur3取hashes[0*4]，
再对{0，1}取murmur3取hashes[1*4],
再对{0，1，2}取murmur3取hashes[2*4],
...
再对{0，1，2,...,n-1}取murmur3取hashes[(n-1)*4],最后以hashes数组为key值进行取hash.

//-----------------------------------------------------------------------------
// This should hopefully be a thorough and uambiguous test of whether a hash
// is correctly implemented on a given platform
//hash是否在给定平台上正确实现,对其进行彻底且明确的测试。
bool VerificationTest ( pfHash hash, const int hashbits, uint32_t expected, bool verbose )
{const int hashbytes = hashbits / 8;uint8_t * key    = new uint8_t[256];uint8_t * hashes = new uint8_t[hashbytes * 256];uint8_t * final  = new uint8_t[hashbytes];memset(key,0,256);memset(hashes,0,hashbytes*256);memset(final,0,hashbytes);// Hash keys of the form {0}, {0,1}, {0,1,2}... up to N=255,using 256-N as the seedfor(int i = 0; i < 256; i++){key[i] = (uint8_t)i;//{0}, {0,1}, {0,1,2}...{0,1,...,255}// 0  , 1   ,  2     ...255//256 , 255 , 254    ...1//0  , 4*1  , 4*2    ...4*255hash(key,i,256-i,&hashes[i*hashbytes]);}// Then hash the result arrayhash(hashes,hashbytes*256,0,final);// The first four bytes of that hash, interpreted as a little-endian integer, is our// verification valueuint32_t verification = (final[0] << 0) | (final[1] << 8) | (final[2] << 16) | (final[3] << 24);delete [] key;delete [] hashes;delete [] final;//----------if(expected != verification){if(verbose) printf("Verification value 0x%08X : Failed! (Expected 0x%08x)\n",verification,expected);return false;}else{if(verbose) printf("Verification value 0x%08X : Passed!\n",verification);return true;}
}

指定Murmur3的hash算法调用

HashInfo * g_hashUnderTest = NULL;void VerifyHash ( const void * key, int len, uint32_t seed, void * out )
{//g_inputVCode = MurmurOAAT(key,len,g_inputVCode);//printf("hash key g_inputVCode: %u ...\n",g_inputVCode);//g_inputVCode = MurmurOAAT(&seed,sizeof(uint32_t *),g_inputVCode);//printf("hash seed g_inputVCode: %u ...\n",g_inputVCode);g_hashUnderTest->hash(key,len,seed,out);printf("hash key hash out: %u ...\n",(uint32_t *)out);//g_outputVCode = MurmurOAAT(out,g_hashUnderTest->hashbits/8,g_outputVCode);//printf("hash out g_outputVCode: %u ...\n",g_outputVCode);
}