concurrent_hash_map.h

/*
    Copyright 2005-2010 Intel Corporation.  All Rights Reserved.

    The source code contained or described herein and all documents related
    to the source code ("Material") are owned by Intel Corporation or its
    suppliers or licensors.  Title to the Material remains with Intel
    Corporation or its suppliers and licensors.  The Material is protected
    by worldwide copyright laws and treaty provisions.  No part of the
    Material may be used, copied, reproduced, modified, published, uploaded,
    posted, transmitted, distributed, or disclosed in any way without
    Intel's prior express written permission.

    No license under any patent, copyright, trade secret or other
    intellectual property right is granted to or conferred upon you by
    disclosure or delivery of the Materials, either expressly, by
    implication, inducement, estoppel or otherwise.  Any license under such
    intellectual property rights must be express and approved by Intel in
    writing.
*/

#ifndef __TBB_concurrent_hash_map_H
#define __TBB_concurrent_hash_map_H

#include "tbb_stddef.h"

#if !TBB_USE_EXCEPTIONS && _MSC_VER
    // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers
    #pragma warning (push)
    #pragma warning (disable: 4530)
#endif

#include <iterator>
#include <utility>      // Need std::pair
#include <cstring>      // Need std::memset

#if !TBB_USE_EXCEPTIONS && _MSC_VER
    #pragma warning (pop)
#endif

#include "cache_aligned_allocator.h"
#include "tbb_allocator.h"
#include "spin_rw_mutex.h"
#include "atomic.h"
#include "aligned_space.h"
#include "tbb_exception.h"
#include "_concurrent_unordered_internal.h" // Need tbb_hasher
#if TBB_USE_PERFORMANCE_WARNINGS
#include <typeinfo>
#endif

namespace tbb {

namespace internal {
    void* __TBB_EXPORTED_FUNC itt_load_pointer_with_acquire_v3( const void* src );
    void __TBB_EXPORTED_FUNC itt_store_pointer_with_release_v3( void* dst, void* src );
    void* __TBB_EXPORTED_FUNC itt_load_pointer_v3( const void* src );
}

template<typename Key>
struct tbb_hash_compare {
    static size_t hash( const Key& a ) { return tbb_hasher(a); }
    static bool equal( const Key& a, const Key& b ) { return a == b; }
};

namespace interface4 {

    template<typename Key, typename T, typename HashCompare = tbb_hash_compare<Key>, typename A = tbb_allocator<std::pair<Key, T> > >
    class concurrent_hash_map;

    namespace internal {


    typedef size_t hashcode_t;
    struct hash_map_node_base : tbb::internal::no_copy {
        typedef spin_rw_mutex mutex_t;
        typedef mutex_t::scoped_lock scoped_t;
        hash_map_node_base *next;
        mutex_t mutex;
    };
    static hash_map_node_base *const rehash_req = reinterpret_cast<hash_map_node_base*>(size_t(3));
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );
            cache_aligned_allocator<bucket> alloc;
            size_type sz;
            __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
            if( k >= first_block ) {
                sz = segment_size( k );
    static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
    class hash_map_base {
    public:
        typedef size_t size_type;
        typedef size_t hashcode_t;
        typedef size_t segment_index_t;
        typedef hash_map_node_base node_base;
        struct bucket : tbb::internal::no_copy {
            typedef spin_rw_mutex mutex_t;
            typedef mutex_t::scoped_lock scoped_t;
            mutex_t mutex;
            node_base *node_list;
        };
        static size_type const embedded_block = 1;
        static size_type const embedded_buckets = 1<<embedded_block;
        static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
        static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
        typedef bucket *segment_ptr_t;
        typedef segment_ptr_t segments_table_t[pointers_per_table];
        atomic<hashcode_t> my_mask;
        segments_table_t my_table;
        atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
        bucket my_embedded_segment[embedded_buckets];

        hash_map_base() {
            std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
                + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
                + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
            for( size_type i = 0; i < embedded_block; i++ ) // fill the table
                my_table[i] = my_embedded_segment + segment_base(i);
            my_mask = embedded_buckets - 1;
            __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
        }

        static segment_index_t segment_index_of( size_type index ) {
            return segment_index_t( __TBB_Log2( index|1 ) );
        }

        static segment_index_t segment_base( segment_index_t k ) {
            return (segment_index_t(1)<<k & ~segment_index_t(1));
        }

        static size_type segment_size( segment_index_t k ) {
            return size_type(1)<<k; // fake value for k==0
        }
        
        static bool is_valid( void *ptr ) {
            return reinterpret_cast<size_t>(ptr) > size_t(63);
        }

        static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
            if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
            else for(size_type i = 0; i < sz; i++, ptr++) {
                    *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
                    ptr->node_list = rehash_req;
                }
        }
        
        static void add_to_bucket( bucket *b, node_base *n ) {
            __TBB_ASSERT(b->node_list != rehash_req, NULL);
            n->next = b->node_list;
            b->node_list = n; // its under lock and flag is set
        }

        struct enable_segment_failsafe {
            segment_ptr_t *my_segment_ptr;
            enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
            ~enable_segment_failsafe() {
                if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
            }
        };

        void enable_segment( segment_index_t k, bool is_initial = false ) {
            __TBB_ASSERT( k, "Zero segment must be embedded" );
            enable_segment_failsafe watchdog( my_table, k );