From 9d2f2afd5f97aefbbebeee89e210ec9a8ea80032 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Tue, 4 Dec 2018 19:23:12 +0300 Subject: [PATCH 01/29] Init ts_deque --- cds/container/ts_deque.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 cds/container/ts_deque.h diff --git a/cds/container/ts_deque.h b/cds/container/ts_deque.h new file mode 100644 index 000000000..8b144f13e --- /dev/null +++ b/cds/container/ts_deque.h @@ -0,0 +1,25 @@ +template +class TSDeque { + private: + + public: + TSDeque (uint64_t num_threads, uint64_t delay) + { + } + + bool insert_left(T element) + { + } + + bool insert_right(T element) + { + } + + bool remove_left(T *element) + { + } + + bool remove_right(T *element) + { + } +}; From 4e31819402da244a18584d3555e8c596d94f8641 Mon Sep 17 00:00:00 2001 From: EduardBlees Date: Thu, 6 Dec 2018 21:44:43 +0300 Subject: [PATCH 02/29] init deque_buffer --- cds/container/ts_deque_buffer.h | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 cds/container/ts_deque_buffer.h diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h new file mode 100644 index 000000000..7d2624fa5 --- /dev/null +++ b/cds/container/ts_deque_buffer.h @@ -0,0 +1,34 @@ +template +class TSDequeBuffer { + private: + + // Returns the leftmost not-taken item from the thread-local list + // indicated by thread_id. + Item* get_left_item(uint64_t thread_id) { + + } + + // Returns the rightmost not-taken item from the thread-local list + // indicated by thread_id. + Item* get_right_item(uint64_t thread_id) { + + } + + public: + + inline std::atomic *insert_left(T element) { + + } + + inline std::atomic *insert_right(T element) { + + } + + bool try_remove_left(T *element, uint64_t *invocation_time) { + + } + + bool try_remove_right(T *element, uint64_t *invocation_time) { + + } +}; From c20d3c300ae059e5d626d9c8c8808f7f9fd2a0b6 Mon Sep 17 00:00:00 2001 From: vikkiorrikki Date: Thu, 6 Dec 2018 22:24:12 +0300 Subject: [PATCH 03/29] init timestamp --- cds/container/ts_timestamp.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 cds/container/ts_timestamp.h diff --git a/cds/container/ts_timestamp.h b/cds/container/ts_timestamp.h new file mode 100644 index 000000000..f0891233c --- /dev/null +++ b/cds/container/ts_timestamp.h @@ -0,0 +1,23 @@ +class HardwareTimestamp { + public: + inline void initialize(uint64_t delay, uint64_t num_threads) { + } + + inline void init_sentinel(uint64_t *result) { + } + + inline void init_top(uint64_t *result) { + } + + inline void load_timestamp(uint64_t *result, std::atomic *source) { + } + + inline void set_timestamp(std::atomic *result) { + } + + inline void read_time(uint64_t *result) { + } + + inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) { + } +}; From 86ff17ddc953f9a94d4612aa1044b576179f5260 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Sat, 8 Dec 2018 23:11:12 +0300 Subject: [PATCH 04/29] add Item and initialize method --- cds/container/ts_deque_buffer.h | 65 ++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index 7d2624fa5..86d925fd6 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -2,6 +2,28 @@ template class TSDequeBuffer { private: + typedef struct Item { + std::atomic left; + std::atomic right; + std::atomic taken; + std::atomic data; + std::atomic timestamp[2]; + // Insertion index, needed for the termination condition in + // get_left_item. Items inserted at the left get negative + // indices, items inserted at the right get positive indices. + std::atomic index; + } Item; + + // The number of threads. + uint64_t num_threads_; + std::atomic **left_; + std::atomic **right_; + int64_t **next_index_; + // The pointers for the emptiness check. + Item** *emptiness_check_left_; + Item** *emptiness_check_right_; + TimeStamp *timestamping_; + // Returns the leftmost not-taken item from the thread-local list // indicated by thread_id. Item* get_left_item(uint64_t thread_id) { @@ -16,6 +38,47 @@ class TSDequeBuffer { public: + void initialize(uint64_t num_threads, TimeStamp *timestamping) { + + num_threads_ = num_threads; + timestamping_ = timestamping; + + left_ = new std::atomic*[num_threads_]; + + right_ = new std::atomic*[num_threads_]; + + next_index_ = new int64_t*[num_threads_]; + + emptiness_check_left_ = new Item**[num_threads_]; + + emptiness_check_right_ = new Item**[num_threads_]; + + for (uint64_t i = 0; i < num_threads_; i++) { + + left_[i] = new std::atomic(); + + right_[i] = new std::atomic(); + + next_index_[i] = new int64_t(); + + // Add a sentinal node. + Item *new_item = new Item(); + timestamping_->init_sentinel_atomic(new_item->timestamp); + new_item->data.store(0); + new_item->taken.store(1); + new_item->left.store(new_item); + new_item->right.store(new_item); + new_item->index.store(0); + left_[i]->store(new_item); + right_[i]->store(new_item); + *next_index_[i] = 1; + + emptiness_check_left_[i] = new Item*[num_threads_]; + + emptiness_check_right_[i] = new Item*[num_threads_]; + } + } + inline std::atomic *insert_left(T element) { } @@ -29,6 +92,6 @@ class TSDequeBuffer { } bool try_remove_right(T *element, uint64_t *invocation_time) { - + } }; From c7c96bf106f39e5c7b8c305978e41e0c4a1c3aa7 Mon Sep 17 00:00:00 2001 From: EduardBlees Date: Mon, 10 Dec 2018 23:13:44 +0300 Subject: [PATCH 05/29] try remove left impl --- cds/container/ts_deque_buffer.h | 308 ++++++++++++++++++++++---------- 1 file changed, 215 insertions(+), 93 deletions(-) diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index 86d925fd6..d7e2dc03c 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -1,97 +1,219 @@ -template -class TSDequeBuffer { +template +class TSDequeBuffer +{ private: - - typedef struct Item { - std::atomic left; - std::atomic right; - std::atomic taken; - std::atomic data; - std::atomic timestamp[2]; - // Insertion index, needed for the termination condition in - // get_left_item. Items inserted at the left get negative - // indices, items inserted at the right get positive indices. - std::atomic index; - } Item; - - // The number of threads. - uint64_t num_threads_; - std::atomic **left_; - std::atomic **right_; - int64_t **next_index_; - // The pointers for the emptiness check. - Item** *emptiness_check_left_; - Item** *emptiness_check_right_; - TimeStamp *timestamping_; - - // Returns the leftmost not-taken item from the thread-local list - // indicated by thread_id. - Item* get_left_item(uint64_t thread_id) { - - } - - // Returns the rightmost not-taken item from the thread-local list - // indicated by thread_id. - Item* get_right_item(uint64_t thread_id) { - - } + typedef struct Item + { + std::atomic left; + std::atomic right; + std::atomic taken; + std::atomic data; + std::atomic timestamp[2]; + // Insertion index, needed for the termination condition in + // get_left_item. Items inserted at the left get negative + // indices, items inserted at the right get positive indices. + std::atomic index; + } Item; + + // The number of threads. + uint64_t num_threads_; + std::atomic **left_; + std::atomic **right_; + int64_t **next_index_; + // The pointers for the emptiness check. + Item ***emptiness_check_left_; + Item ***emptiness_check_right_; + TimeStamp *timestamping_; + + // Returns the leftmost not-taken item from the thread-local list + // indicated by thread_id. + Item *get_left_item(uint64_t thread_id) + { + } + + // Returns the rightmost not-taken item from the thread-local list + // indicated by thread_id. + Item *get_right_item(uint64_t thread_id) + { + } public: - - void initialize(uint64_t num_threads, TimeStamp *timestamping) { - - num_threads_ = num_threads; - timestamping_ = timestamping; - - left_ = new std::atomic*[num_threads_]; - - right_ = new std::atomic*[num_threads_]; - - next_index_ = new int64_t*[num_threads_]; - - emptiness_check_left_ = new Item**[num_threads_]; - - emptiness_check_right_ = new Item**[num_threads_]; - - for (uint64_t i = 0; i < num_threads_; i++) { - - left_[i] = new std::atomic(); - - right_[i] = new std::atomic(); - - next_index_[i] = new int64_t(); - - // Add a sentinal node. - Item *new_item = new Item(); - timestamping_->init_sentinel_atomic(new_item->timestamp); - new_item->data.store(0); - new_item->taken.store(1); - new_item->left.store(new_item); - new_item->right.store(new_item); - new_item->index.store(0); - left_[i]->store(new_item); - right_[i]->store(new_item); - *next_index_[i] = 1; - - emptiness_check_left_[i] = new Item*[num_threads_]; - - emptiness_check_right_[i] = new Item*[num_threads_]; - } - } - - inline std::atomic *insert_left(T element) { - - } - - inline std::atomic *insert_right(T element) { - - } - - bool try_remove_left(T *element, uint64_t *invocation_time) { - - } - - bool try_remove_right(T *element, uint64_t *invocation_time) { - - } + void initialize(uint64_t num_threads, TimeStamp *timestamping) + { + + num_threads_ = num_threads; + timestamping_ = timestamping; + + left_ = new std::atomic *[num_threads_]; + + right_ = new std::atomic *[num_threads_]; + + next_index_ = new int64_t *[num_threads_]; + + emptiness_check_left_ = new Item **[num_threads_]; + + emptiness_check_right_ = new Item **[num_threads_]; + + for (uint64_t i = 0; i < num_threads_; i++) + { + + left_[i] = new std::atomic(); + + right_[i] = new std::atomic(); + + next_index_[i] = new int64_t(); + + // Add a sentinal node. + Item *new_item = new Item(); + timestamping_->init_sentinel_atomic(new_item->timestamp); + new_item->data.store(0); + new_item->taken.store(1); + new_item->left.store(new_item); + new_item->right.store(new_item); + new_item->index.store(0); + left_[i]->store(new_item); + right_[i]->store(new_item); + *next_index_[i] = 1; + + emptiness_check_left_[i] = new Item *[num_threads_]; + + emptiness_check_right_[i] = new Item *[num_threads_]; + } + } + + inline std::atomic *insert_left(T element) + { + } + + inline std::atomic *insert_right(T element) + { + } + + bool try_remove_left(T *element, uint64_t *invocation_time) + { + // Initialize the data needed for the emptiness check. + uint64_t thread_id = scal::ThreadContext::get().thread_id(); + Item **emptiness_check_left = + emptiness_check_left_[thread_id]; + Item **emptiness_check_right = + emptiness_check_right_[thread_id]; + bool empty = true; + // Initialize the result pointer to NULL, which means that no + // element has been removed. + Item *result = NULL; + // Indicates the index which contains the youngest item. + uint64_t buffer_index = -1; + // Memory on the stack frame where timestamps of items can be stored + // temporarily. + uint64_t tmp_timestamp[2][2]; + // Index in the tmp_timestamp array which is not used at the moment. + uint64_t tmp_index = 1; + timestamping_->init_sentinel(tmp_timestamp[0]); + uint64_t *timestamp = tmp_timestamp[0]; + // Stores the value of the remove pointer of a thead-local buffer + // before the buffer is actually accessed. + Item *old_left = NULL; + + // Read the start time of the iteration. Items which were timestamped + // after the start time and inserted at the right are not removed. + uint64_t start_time[2]; + timestamping_->read_time(start_time); + // We start iterating over the thread-local lists at a random index. + uint64_t start = hwrand(); + // We iterate over all thead-local buffers + for (uint64_t i = 0; i < num_threads_; i++) + { + uint64_t tmp_buffer_index = (start + i) % num_threads_; + // We get the remove/insert pointer of the current thread-local buffer. + Item *tmp_left = left_[tmp_buffer_index]->load(); + // We get the youngest element from that thread-local buffer. + Item *item = get_left_item(tmp_buffer_index); + // If we found an element, we compare it to the youngest element + // we have found until now. + if (item != NULL) + { + empty = false; + uint64_t *item_timestamp; + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + + if (inserted_left(item) && !timestamping_->is_later(invocation_time, item_timestamp)) + { + uint64_t expected = 0; + if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + left_[tmp_buffer_index]->compare_exchange_weak( + tmp_left, (Item *)add_next_aba(item, tmp_left, 0)); + *element = item->data.load(); + return true; + } + else + { + item = get_left_item(tmp_buffer_index); + if (item != NULL) + { + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + } + } + } + + if (item != NULL && (result == NULL || is_more_left(item, item_timestamp, result, timestamp))) + { + // We found a new leftmost item, so we remember it. + result = item; + buffer_index = tmp_buffer_index; + timestamp = item_timestamp; + tmp_index ^= 1; + old_left = tmp_left; + + // Check if we can remove the element immediately. + if (inserted_left(result) && !timestamping_->is_later(invocation_time, timestamp)) + { + uint64_t expected = 0; + if (result->taken.load() == 0) + { + if (result->taken.compare_exchange_weak( + expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + left_[buffer_index]->compare_exchange_weak( + old_left, (Item *)add_next_aba(result, old_left, 0)); + + *element = result->data.load(); + return true; + } + } + } + } + } + else + { + // No element was found, work on the emptiness check. + if (emptiness_check_left[tmp_buffer_index] != tmp_left) + { + empty = false; + emptiness_check_left[tmp_buffer_index] = + tmp_left; + } + Item *tmp_right = right_[tmp_buffer_index]->load(); + if (emptiness_check_right[tmp_buffer_index] != tmp_right) + { + empty = false; + emptiness_check_right[tmp_buffer_index] = + tmp_right; + } + } + } + + *element = (T)NULL; + return !empty; + } + + bool try_remove_right(T *element, uint64_t *invocation_time) + { + } }; From 9f6d53ef31c178ff1fbe73eec88dbff442f9d6f3 Mon Sep 17 00:00:00 2001 From: EduardBlees Date: Mon, 10 Dec 2018 23:34:45 +0300 Subject: [PATCH 06/29] threadcontext --- cds/container/threadcontext.h | 41 +++++++++++++++++++++++++++++++++ cds/container/ts_deque_buffer.h | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 cds/container/threadcontext.h diff --git a/cds/container/threadcontext.h b/cds/container/threadcontext.h new file mode 100644 index 000000000..4a4f10f7b --- /dev/null +++ b/cds/container/threadcontext.h @@ -0,0 +1,41 @@ +#include +#include + +class ThreadContext { + public: + static ThreadContext& get(); + static void prepare(uint64_t num_threads); + + inline uint64_t thread_id() { + return thread_id_; + } + + private: + static constexpr uint64_t kMaxThreads = 1024; + static uint64_t global_thread_id_cnt; + static ThreadContext *contexts[kMaxThreads]; + static pthread_key_t threadcontext_key; + + ThreadContext() {} + + uint64_t thread_id_; +}; + +uint64_t ThreadContext::global_thread_id_cnt = 0; +pthread_key_t ThreadContext::threadcontext_key; +ThreadContext *ThreadContext::contexts[kMaxThreads]; + +ThreadContext& ThreadContext::get() { + ThreadContext *context = static_cast( + pthread_getspecific(threadcontext_key)); + return *context; +} + +void ThreadContext::prepare(uint64_t num_threads) { + pthread_key_create(&threadcontext_key, NULL); + for (uint64_t i = 0; i < num_threads; i++) { + ThreadContext *context = new ThreadContext(); + context->thread_id_ = i; + contexts[i] = context; + } +} \ No newline at end of file diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index d7e2dc03c..3e449935b 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -92,7 +92,7 @@ class TSDequeBuffer bool try_remove_left(T *element, uint64_t *invocation_time) { // Initialize the data needed for the emptiness check. - uint64_t thread_id = scal::ThreadContext::get().thread_id(); + uint64_t thread_id = ThreadContext::get().thread_id(); Item **emptiness_check_left = emptiness_check_left_[thread_id]; Item **emptiness_check_right = From 92efc844976d9380a977de9842c324b006e802b7 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Wed, 12 Dec 2018 22:12:11 +0300 Subject: [PATCH 07/29] implement insert_right --- cds/container/ts_deque_buffer.h | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index 3e449935b..9c435c385 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -85,8 +85,47 @@ class TSDequeBuffer { } + ///////////////////////////////////////////////////////////////// + // insert_right + ///////////////////////////////////////////////////////////////// inline std::atomic *insert_right(T element) { + uint64_t thread_id = scal::ThreadContext::get().thread_id(); + + // Create a new item. + Item *new_item = new Item(); + timestamping_->init_top_atomic(new_item->timestamp); + new_item->data.store(element); + new_item->taken.store(0); + new_item->right.store(new_item); + new_item->index = (*next_index_[thread_id])++; + + // Determine the rightmost not-taken item in the list. The new item is + // inserted to the right of that item. + Item *old_right = right_[thread_id]->load(); + + Item *right = old_right; + while (right->left.load() != right && right->taken.load()) + { + right = right->left.load(); + } + + if (right->taken.load() && right->left.load() == right) + { + right = old_right; + right->left.store(right); + Item *old_left = left_[thread_id]->load(); + left_[thread_id]->store(right); + } + + // Add the new item to the list. + new_item->left.store(right); + right->right.store(new_item); + right_[thread_id]->store(new_item); + + // Return a pointer to the timestamp location of the item so that a + // timestamp can be added. + return new_item->timestamp; } bool try_remove_left(T *element, uint64_t *invocation_time) From a136c71ce6b3265cb6da0b26f966ca7ae2f0cbe1 Mon Sep 17 00:00:00 2001 From: vikkiorrikki Date: Wed, 12 Dec 2018 23:35:03 +0300 Subject: [PATCH 08/29] timestamp hardware --- cds/container/ts_timestamp.h | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cds/container/ts_timestamp.h b/cds/container/ts_timestamp.h index f0891233c..99992f9a3 100644 --- a/cds/container/ts_timestamp.h +++ b/cds/container/ts_timestamp.h @@ -1,23 +1,45 @@ +inline uint64_t get_hwptime(void) +{ + uint64_t aux; + uint64_t rax,rdx; + asm volatile ( "rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); + return (rdx << 32) + rax; +} + class HardwareTimestamp { public: inline void initialize(uint64_t delay, uint64_t num_threads) { } inline void init_sentinel(uint64_t *result) { + result[0] = 0; + } + + inline void init_sentinel_atomic(std::atomic *result) { + result[0].store(0); + } + + inline void init_top_atomic(std::atomic *result) { + result[0].store(UINT64_MAX); } inline void init_top(uint64_t *result) { + result[0] = UINT64_MAX; } inline void load_timestamp(uint64_t *result, std::atomic *source) { + result[0] = source[0].load(); } inline void set_timestamp(std::atomic *result) { + result[0].store(get_hwptime()); } inline void read_time(uint64_t *result) { + result[0] = get_hwptime(); } inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) { + return timestamp2[0] < timestamp1[0]; } -}; +}; \ No newline at end of file From 0dcef53ba0b4971bab5cefee92095410edf54a9c Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Thu, 13 Dec 2018 23:58:11 +0300 Subject: [PATCH 09/29] insert_left implementation --- cds/container/ts_deque_buffer.h | 47 ++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index 9c435c385..292904bfe 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -81,9 +81,50 @@ class TSDequeBuffer } } - inline std::atomic *insert_left(T element) - { - } + ///////////////////////////////////////////////////////////////// + // insert_right + ///////////////////////////////////////////////////////////////// + inline std::atomic *insert_left(T element) { + uint64_t thread_id = ThreadContext::get().thread_id(); + + // Create a new item. + Item *new_item = new Item(); + timestamping_->init_top_atomic(new_item->timestamp); + new_item->data.store(element); + new_item->taken.store(0); + new_item->left.store(new_item); + // Items inserted at the left get negative indices. Thereby the + // order of items in the thread-local lists correspond with the + // order of indices, and we can use the sign of the index to + // determine on which side an item has been inserted. + new_item->index = -((*next_index_[thread_id])++); + + // Determine leftmost not-taken item in the list. The new item is + // inserted to the left of that item. + Item* old_left = left_[thread_id]->load(); + + Item* left = old_left; + while (left->right.load() != left + && left->taken.load()) { + left = left->right.load(); + } + + if (left->taken.load() && left->right.load() == left) { + left = old_left; + left->right.store(left); + Item* old_right = right_[thread_id]->load(); + right_[thread_id]->store(left); + } + + // Add the new item to the list. + new_item->right.store(left); + left->left.store(new_item); + left_[thread_id]->store(new_item); + + // Return a pointer to the timestamp location of the item so that a + // timestamp can be added. + return new_item->timestamp; + } ///////////////////////////////////////////////////////////////// // insert_right From 7fdf05c48c998594ff41dfe5b18f6df7aefb9614 Mon Sep 17 00:00:00 2001 From: EduardBlees Date: Sat, 15 Dec 2018 20:30:04 +0300 Subject: [PATCH 10/29] try remove right impl --- cds/container/ts_deque_buffer.h | 265 ++++++++++++++++++++++++++------ 1 file changed, 214 insertions(+), 51 deletions(-) diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index 292904bfe..e1ee7d33c 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -81,57 +81,53 @@ class TSDequeBuffer } } - ///////////////////////////////////////////////////////////////// - // insert_right - ///////////////////////////////////////////////////////////////// - inline std::atomic *insert_left(T element) { - uint64_t thread_id = ThreadContext::get().thread_id(); - - // Create a new item. - Item *new_item = new Item(); - timestamping_->init_top_atomic(new_item->timestamp); - new_item->data.store(element); - new_item->taken.store(0); - new_item->left.store(new_item); - // Items inserted at the left get negative indices. Thereby the - // order of items in the thread-local lists correspond with the - // order of indices, and we can use the sign of the index to - // determine on which side an item has been inserted. - new_item->index = -((*next_index_[thread_id])++); - - // Determine leftmost not-taken item in the list. The new item is - // inserted to the left of that item. - Item* old_left = left_[thread_id]->load(); - - Item* left = old_left; - while (left->right.load() != left - && left->taken.load()) { - left = left->right.load(); - } - - if (left->taken.load() && left->right.load() == left) { - left = old_left; - left->right.store(left); - Item* old_right = right_[thread_id]->load(); - right_[thread_id]->store(left); - } - - // Add the new item to the list. - new_item->right.store(left); - left->left.store(new_item); - left_[thread_id]->store(new_item); - - // Return a pointer to the timestamp location of the item so that a - // timestamp can be added. - return new_item->timestamp; - } - - ///////////////////////////////////////////////////////////////// - // insert_right - ///////////////////////////////////////////////////////////////// + inline std::atomic *insert_left(T element) + { + uint64_t thread_id = ThreadContext::get().thread_id(); + + // Create a new item. + Item *new_item = new Item(); + timestamping_->init_top_atomic(new_item->timestamp); + new_item->data.store(element); + new_item->taken.store(0); + new_item->left.store(new_item); + // Items inserted at the left get negative indices. Thereby the + // order of items in the thread-local lists correspond with the + // order of indices, and we can use the sign of the index to + // determine on which side an item has been inserted. + new_item->index = -((*next_index_[thread_id])++); + + // Determine leftmost not-taken item in the list. The new item is + // inserted to the left of that item. + Item *old_left = left_[thread_id]->load(); + + Item *left = old_left; + while (left->right.load() != left && left->taken.load()) + { + left = left->right.load(); + } + + if (left->taken.load() && left->right.load() == left) + { + left = old_left; + left->right.store(left); + Item *old_right = right_[thread_id]->load(); + right_[thread_id]->store(left); + } + + // Add the new item to the list. + new_item->right.store(left); + left->left.store(new_item); + left_[thread_id]->store(new_item); + + // Return a pointer to the timestamp location of the item so that a + // timestamp can be added. + return new_item->timestamp; + } + inline std::atomic *insert_right(T element) { - uint64_t thread_id = scal::ThreadContext::get().thread_id(); + uint64_t thread_id = ThreadContext::get().thread_id(); // Create a new item. Item *new_item = new Item(); @@ -169,6 +165,72 @@ class TSDequeBuffer return new_item->timestamp; } + // Helper function which returns true if the item was inserted at the left. + inline bool inserted_left(Item *item) + { + return item->index.load() < 0; + } + + // Helper function which returns true if the item was inserted at the right. + inline bool inserted_right(Item *item) + { + return item->index.load() > 0; + } + + // Helper function which returns true if item1 is more left than item2. + inline bool is_more_left(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) + { + if (inserted_left(item2)) + { + if (inserted_left(item1)) + { + return timestamping_->is_later(timestamp1, timestamp2); + } + else + { + return false; + } + } + else + { + if (inserted_left(item1)) + { + return true; + } + else + { + return timestamping_->is_later(timestamp2, timestamp1); + } + } + } + + // Helper function which returns true if item1 is more right than item2. + inline bool is_more_right(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) + { + if (inserted_right(item2)) + { + if (inserted_right(item1)) + { + return timestamping_->is_later(timestamp1, timestamp2); + } + else + { + return false; + } + } + else + { + if (inserted_right(item1)) + { + return true; + } + else + { + return timestamping_->is_later(timestamp2, timestamp1); + } + } + } + bool try_remove_left(T *element, uint64_t *invocation_time) { // Initialize the data needed for the emptiness check. @@ -225,7 +287,7 @@ class TSDequeBuffer // Try to adjust the remove pointer. It does not matter if // this CAS fails. left_[tmp_buffer_index]->compare_exchange_weak( - tmp_left, (Item *)add_next_aba(item, tmp_left, 0)); + tmp_left, item); *element = item->data.load(); return true; } @@ -261,7 +323,7 @@ class TSDequeBuffer // Try to adjust the remove pointer. It does not matter if // this CAS fails. left_[buffer_index]->compare_exchange_weak( - old_left, (Item *)add_next_aba(result, old_left, 0)); + old_left, item); *element = result->data.load(); return true; @@ -295,5 +357,106 @@ class TSDequeBuffer bool try_remove_right(T *element, uint64_t *invocation_time) { + // Initialize the data needed for the emptiness check. + uint64_t thread_id = ThreadContext::get().thread_id(); + Item **emptiness_check_left = + emptiness_check_left_[thread_id]; + Item **emptiness_check_right = + emptiness_check_right_[thread_id]; + bool empty = true; + // Initialize the result pointer to NULL, which means that no + // element has been removed. + Item *result = NULL; + // Indicates the index which contains the youngest item. + uint64_t buffer_index = -1; + // Memory on the stack frame where timestamps of items can be stored + // temporarily. + uint64_t tmp_timestamp[2][2]; + // Index in the tmp_timestamp array whihc is not used at the moment. + uint64_t tmp_index = 1; + timestamping_->init_sentinel(tmp_timestamp[0]); + uint64_t *timestamp = tmp_timestamp[0]; + // Stores the value of the remove pointer of a thead-local buffer + // before the buffer is actually accessed. + Item *old_right = NULL; + + // Read the start time of the iteration. Items which were timestamped + // after the start time and inserted at the left are not removed. + uint64_t start_time[2]; + timestamping_->read_time(start_time); + // We start iterating over the thread-local lists at a random index. + uint64_t start = hwrand(); + // We iterate over all thead-local buffers + for (uint64_t i = 0; i < num_threads_; i++) + { + + uint64_t tmp_buffer_index = (start + i) % num_threads_; + // We get the remove/insert pointer of the current thread-local buffer. + Item *tmp_right = right_[tmp_buffer_index]->load(); + // We get the youngest element from that thread-local buffer. + Item *item = get_right_item(tmp_buffer_index); + // If we found an element, we compare it to the youngest element + // we have found until now. + if (item != NULL) + { + empty = false; + uint64_t *item_timestamp; + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + + if (inserted_right(item) && !timestamping_->is_later(invocation_time, item_timestamp)) + { + uint64_t expected = 0; + if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + right_[tmp_buffer_index]->compare_exchange_weak( + tmp_right, item); + *element = item->data.load(); + return true; + } + else + { + item = get_right_item(tmp_buffer_index); + if (item != NULL) + { + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + } + } + } + + if (item != NULL && (result == NULL || is_more_right(item, item_timestamp, result, timestamp))) + { + // We found a new youngest element, so we remember it. + result = item; + buffer_index = tmp_buffer_index; + timestamp = item_timestamp; + tmp_index ^= 1; + old_right = tmp_right; + } + } + else + { + // No element was found, work on the emptiness check. + if (emptiness_check_right[tmp_buffer_index] != tmp_right) + { + empty = false; + emptiness_check_right[tmp_buffer_index] = + tmp_right; + } + Item *tmp_left = left_[tmp_buffer_index]->load(); + if (emptiness_check_left[tmp_buffer_index] != tmp_left) + { + empty = false; + emptiness_check_left[tmp_buffer_index] = + tmp_left; + } + } + } + + *element = (T)NULL; + return !empty; } }; From ab723b5c2fa4ca1680de931f40038a6a0f790397 Mon Sep 17 00:00:00 2001 From: vikkiorrikki Date: Sat, 15 Dec 2018 23:24:16 +0300 Subject: [PATCH 11/29] timestamp hardware interval --- cds/container/ts_timestamp.h | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/cds/container/ts_timestamp.h b/cds/container/ts_timestamp.h index 99992f9a3..0ea907a80 100644 --- a/cds/container/ts_timestamp.h +++ b/cds/container/ts_timestamp.h @@ -6,6 +6,12 @@ inline uint64_t get_hwptime(void) return (rdx << 32) + rax; } +inline uint64_t get_hwtime(void) { + unsigned int hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t) lo) | (((uint64_t) hi) << 32); +} + class HardwareTimestamp { public: inline void initialize(uint64_t delay, uint64_t num_threads) { @@ -42,4 +48,55 @@ class HardwareTimestamp { inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) { return timestamp2[0] < timestamp1[0]; } +}; + +class HardwareIntervalTimestamp { + private: + uint64_t delay_; + + public: + inline void initialize(uint64_t delay, uint64_t num_threads) { + delay_ = delay; + } + + inline void init_sentinel(uint64_t *result) { + result[0] = 0; + result[1] = 0; + } + + inline void init_sentinel_atomic(std::atomic *result) { + result[0].store(0); + result[1].store(0); + } + + inline void init_top_atomic(std::atomic *result) { + result[0].store(UINT64_MAX); + result[1].store(UINT64_MAX); + } + + inline void init_top(uint64_t *result) { + result[0] = UINT64_MAX; + result[1] = UINT64_MAX; + } + + inline void load_timestamp(uint64_t *result, std::atomic *source) { + result[0] = source[0].load(); + result[1] = source[1].load(); + } + + inline void set_timestamp(std::atomic *result) { + result[0].store(get_hwptime()); + uint64_t wait = get_hwtime() + delay_; + while (get_hwtime() < wait) {} + result[1].store(get_hwptime()); + } + + inline void read_time(uint64_t *result) { + result[0] = get_hwptime(); + result[1] = result[0]; + } + + inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) { + return timestamp2[1] < timestamp1[0]; + } }; \ No newline at end of file From b2a8b91b9bfda01834bef910efa6ba5de71107c6 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Sun, 16 Dec 2018 20:06:11 +0300 Subject: [PATCH 12/29] implement ts_deque --- cds/container/ts_deque.h | 49 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/cds/container/ts_deque.h b/cds/container/ts_deque.h index 8b144f13e..6aa9f82fc 100644 --- a/cds/container/ts_deque.h +++ b/cds/container/ts_deque.h @@ -1,25 +1,72 @@ +#include +#include "ts_deque_buffer.h" + template class TSDeque { private: + TSDequeBuffer *buffer_; + Timestamp *timestamping_; + public: - TSDeque (uint64_t num_threads, uint64_t delay) + TSDeque (uint64_t num_threads) { + + timestamping_ = new Timestamp(); + + buffer_ = new TSDequeBuffer(); + buffer_->initialize(num_threads, timestamping_); } bool insert_left(T element) { + std::atomic *item = buffer_->insert_left(element); + // In the set_timestamp operation first a new timestamp is acquired + // and then assigned to the item. The operation may not be executed + // atomically. + timestamping_->set_timestamp(item); + return true; } bool insert_right(T element) { + std::atomic *item = buffer_->insert_right(element); + // In the set_timestamp operation first a new timestamp is acquired + // and then assigned to the item. The operation may not be executed + // atomically. + timestamping_->set_timestamp(item); + return true; } bool remove_left(T *element) { + // Read the invocation time of this operation, needed for the + // elimination optimization. + uint64_t invocation_time[2]; + timestamping_->read_time(invocation_time); + while (buffer_->try_remove_left(element, invocation_time)) { + + if (*element != (T)NULL) { + return true; + } + } + // The deque was empty, return false. + return false; } bool remove_right(T *element) { + // Read the invocation time of this operation, needed for the + // elimination optimization. + uint64_t invocation_time[2]; + timestamping_->read_time(invocation_time); + while (buffer_->try_remove_right(element, invocation_time)) { + + if (*element != (T)NULL) { + return true; + } + } + // The deque was empty, return false. + return false; } }; From 0178ecb6282299851bb5fbefb5e4b9e578450c2e Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Sun, 16 Dec 2018 23:30:17 +0300 Subject: [PATCH 13/29] methods for fixing aba problem added --- cds/container/ts_deque_buffer.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index e1ee7d33c..c6690ecbb 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -1,3 +1,8 @@ +#include +#include +#include +#include "threadcontext.h" + template class TSDequeBuffer { @@ -25,6 +30,26 @@ class TSDequeBuffer Item ***emptiness_check_right_; TimeStamp *timestamping_; + // Helper function to remove the ABA counter from a pointer. + void *get_aba_free_pointer(void *pointer) + { + uint64_t result = (uint64_t)pointer; + result &= 0xfffffffffffffff8; + return (void *)result; + } + + // Helper function which retrieves the ABA counter of a pointer old + // and sets this ABA counter + increment to the pointer pointer. + void *add_next_aba(void *pointer, void *old, uint64_t increment) + { + uint64_t aba = (uint64_t)old; + aba += increment; + aba &= 0x7; + uint64_t result = (uint64_t)pointer; + result = (result & 0xfffffffffffffff8) | aba; + return (void *)((result & 0xffffffffffffff8) | aba); + } + // Returns the leftmost not-taken item from the thread-local list // indicated by thread_id. Item *get_left_item(uint64_t thread_id) From 2bb8dce292fcab505a99b6d41c8d999fcf98f070 Mon Sep 17 00:00:00 2001 From: EduardBlees Date: Mon, 17 Dec 2018 23:09:02 +0300 Subject: [PATCH 14/29] try remove fixes, use ABA functions --- cds/container/ts_deque_buffer.h | 462 +++++++++++++++++--------------- 1 file changed, 243 insertions(+), 219 deletions(-) diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index c6690ecbb..5f3e6a7a8 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -3,6 +3,16 @@ #include #include "threadcontext.h" +inline uint64_t rdtsc() { + unsigned int hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t) lo) | (((uint64_t) hi) << 32); +} + +inline uint64_t hwrand() { + return (rdtsc() >> 6); +} + template class TSDequeBuffer { @@ -259,229 +269,243 @@ class TSDequeBuffer bool try_remove_left(T *element, uint64_t *invocation_time) { // Initialize the data needed for the emptiness check. - uint64_t thread_id = ThreadContext::get().thread_id(); - Item **emptiness_check_left = - emptiness_check_left_[thread_id]; - Item **emptiness_check_right = - emptiness_check_right_[thread_id]; - bool empty = true; - // Initialize the result pointer to NULL, which means that no - // element has been removed. - Item *result = NULL; - // Indicates the index which contains the youngest item. - uint64_t buffer_index = -1; - // Memory on the stack frame where timestamps of items can be stored - // temporarily. - uint64_t tmp_timestamp[2][2]; - // Index in the tmp_timestamp array which is not used at the moment. - uint64_t tmp_index = 1; - timestamping_->init_sentinel(tmp_timestamp[0]); - uint64_t *timestamp = tmp_timestamp[0]; - // Stores the value of the remove pointer of a thead-local buffer - // before the buffer is actually accessed. - Item *old_left = NULL; - - // Read the start time of the iteration. Items which were timestamped - // after the start time and inserted at the right are not removed. - uint64_t start_time[2]; - timestamping_->read_time(start_time); - // We start iterating over the thread-local lists at a random index. - uint64_t start = hwrand(); - // We iterate over all thead-local buffers - for (uint64_t i = 0; i < num_threads_; i++) - { - uint64_t tmp_buffer_index = (start + i) % num_threads_; - // We get the remove/insert pointer of the current thread-local buffer. - Item *tmp_left = left_[tmp_buffer_index]->load(); - // We get the youngest element from that thread-local buffer. - Item *item = get_left_item(tmp_buffer_index); - // If we found an element, we compare it to the youngest element - // we have found until now. - if (item != NULL) - { - empty = false; - uint64_t *item_timestamp; - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; - - if (inserted_left(item) && !timestamping_->is_later(invocation_time, item_timestamp)) - { - uint64_t expected = 0; - if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) - { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - left_[tmp_buffer_index]->compare_exchange_weak( - tmp_left, item); - *element = item->data.load(); - return true; - } - else - { - item = get_left_item(tmp_buffer_index); - if (item != NULL) - { - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; - } - } - } - - if (item != NULL && (result == NULL || is_more_left(item, item_timestamp, result, timestamp))) - { - // We found a new leftmost item, so we remember it. - result = item; - buffer_index = tmp_buffer_index; - timestamp = item_timestamp; - tmp_index ^= 1; - old_left = tmp_left; - - // Check if we can remove the element immediately. - if (inserted_left(result) && !timestamping_->is_later(invocation_time, timestamp)) - { - uint64_t expected = 0; - if (result->taken.load() == 0) - { - if (result->taken.compare_exchange_weak( - expected, 1)) - { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - left_[buffer_index]->compare_exchange_weak( - old_left, item); - - *element = result->data.load(); - return true; - } - } - } - } - } - else - { - // No element was found, work on the emptiness check. - if (emptiness_check_left[tmp_buffer_index] != tmp_left) - { - empty = false; - emptiness_check_left[tmp_buffer_index] = - tmp_left; - } - Item *tmp_right = right_[tmp_buffer_index]->load(); - if (emptiness_check_right[tmp_buffer_index] != tmp_right) - { - empty = false; - emptiness_check_right[tmp_buffer_index] = - tmp_right; - } - } - } - - *element = (T)NULL; - return !empty; + uint64_t thread_id = scal::ThreadContext::get().thread_id(); + Item* *emptiness_check_left = + emptiness_check_left_[thread_id]; + Item* *emptiness_check_right = + emptiness_check_right_[thread_id]; + bool empty = true; + // Initialize the result pointer to NULL, which means that no + // element has been removed. + Item *result = NULL; + // Indicates the index which contains the youngest item. + uint64_t buffer_index = -1; + // Memory on the stack frame where timestamps of items can be stored + // temporarily. + uint64_t tmp_timestamp[2][2]; + // Index in the tmp_timestamp array which is not used at the moment. + uint64_t tmp_index = 1; + timestamping_->init_sentinel(tmp_timestamp[0]); + uint64_t *timestamp = tmp_timestamp[0]; + // Stores the value of the remove pointer of a thead-local buffer + // before the buffer is actually accessed. + Item* old_left = NULL; + + // Read the start time of the iteration. Items which were timestamped + // after the start time and inserted at the right are not removed. + uint64_t start_time[2]; + timestamping_->read_time(start_time); + // We start iterating over the thread-local lists at a random index. + uint64_t start = hwrand(); + // We iterate over all thead-local buffers + for (uint64_t i = 0; i < num_threads_; i++) { + + uint64_t tmp_buffer_index = (start + i) % num_threads_; + // We get the remove/insert pointer of the current thread-local buffer. + Item* tmp_left = left_[tmp_buffer_index]->load(); + // We get the youngest element from that thread-local buffer. + Item* item = get_left_item(tmp_buffer_index); + // If we found an element, we compare it to the youngest element + // we have found until now. + if (item != NULL) { + empty = false; + uint64_t *item_timestamp; + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + + if (inserted_left(item) && !timestamping_->is_later(invocation_time, item_timestamp)) { + uint64_t expected = 0; + if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + left_[tmp_buffer_index]->compare_exchange_weak( + tmp_left, (Item*)add_next_aba(item, tmp_left, 0)); + *element = item->data.load(); + return true; + } else { + item = get_left_item(tmp_buffer_index); + if (item != NULL) { + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + } + } + } + + if (item != NULL && (result == NULL || is_more_left(item, item_timestamp, result, timestamp))) { + // We found a new leftmost item, so we remember it. + result = item; + buffer_index = tmp_buffer_index; + timestamp = item_timestamp; + tmp_index ^=1; + old_left = tmp_left; + + // Check if we can remove the element immediately. + if (inserted_left(result) && !timestamping_->is_later(invocation_time, timestamp)) { + uint64_t expected = 0; + if (result->taken.load() == 0) { + if (result->taken.compare_exchange_weak( + expected, 1)) { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + left_[buffer_index]->compare_exchange_weak( + old_left, (Item*)add_next_aba(result, old_left, 0)); + + *element = result->data.load(); + return true; + } + } + } + } + } else { + // No element was found, work on the emptiness check. + if (emptiness_check_left[tmp_buffer_index] + != tmp_left) { + empty = false; + emptiness_check_left[tmp_buffer_index] = + tmp_left; + } + Item* tmp_right = right_[tmp_buffer_index]->load(); + if (emptiness_check_right[tmp_buffer_index] + != tmp_right) { + empty = false; + emptiness_check_right[tmp_buffer_index] = + tmp_right; + } + } + } + if (result != NULL) { + if (!timestamping_->is_later(timestamp, start_time)) { + // The found item was timestamped after the start of the iteration, + // so it is save to remove it. + uint64_t expected = 0; + if (result->taken.load() == 0) { + if (result->taken.compare_exchange_weak( + expected, 1)) { + // Try to adjust the remove pointer. It does not matter if this + // CAS fails. + left_[buffer_index]->compare_exchange_weak( + old_left, (Item*)add_next_aba(result, old_left, 0)); + *element = result->data.load(); + return true; + } + } + } + } + + *element = (T)NULL; + return !empty; } bool try_remove_right(T *element, uint64_t *invocation_time) { // Initialize the data needed for the emptiness check. - uint64_t thread_id = ThreadContext::get().thread_id(); - Item **emptiness_check_left = - emptiness_check_left_[thread_id]; - Item **emptiness_check_right = - emptiness_check_right_[thread_id]; - bool empty = true; - // Initialize the result pointer to NULL, which means that no - // element has been removed. - Item *result = NULL; - // Indicates the index which contains the youngest item. - uint64_t buffer_index = -1; - // Memory on the stack frame where timestamps of items can be stored - // temporarily. - uint64_t tmp_timestamp[2][2]; - // Index in the tmp_timestamp array whihc is not used at the moment. - uint64_t tmp_index = 1; - timestamping_->init_sentinel(tmp_timestamp[0]); - uint64_t *timestamp = tmp_timestamp[0]; - // Stores the value of the remove pointer of a thead-local buffer - // before the buffer is actually accessed. - Item *old_right = NULL; - - // Read the start time of the iteration. Items which were timestamped - // after the start time and inserted at the left are not removed. - uint64_t start_time[2]; - timestamping_->read_time(start_time); - // We start iterating over the thread-local lists at a random index. - uint64_t start = hwrand(); - // We iterate over all thead-local buffers - for (uint64_t i = 0; i < num_threads_; i++) - { - - uint64_t tmp_buffer_index = (start + i) % num_threads_; - // We get the remove/insert pointer of the current thread-local buffer. - Item *tmp_right = right_[tmp_buffer_index]->load(); - // We get the youngest element from that thread-local buffer. - Item *item = get_right_item(tmp_buffer_index); - // If we found an element, we compare it to the youngest element - // we have found until now. - if (item != NULL) - { - empty = false; - uint64_t *item_timestamp; - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; - - if (inserted_right(item) && !timestamping_->is_later(invocation_time, item_timestamp)) - { - uint64_t expected = 0; - if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) - { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - right_[tmp_buffer_index]->compare_exchange_weak( - tmp_right, item); - *element = item->data.load(); - return true; - } - else - { - item = get_right_item(tmp_buffer_index); - if (item != NULL) - { - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; - } - } - } - - if (item != NULL && (result == NULL || is_more_right(item, item_timestamp, result, timestamp))) - { - // We found a new youngest element, so we remember it. - result = item; - buffer_index = tmp_buffer_index; - timestamp = item_timestamp; - tmp_index ^= 1; - old_right = tmp_right; - } - } - else - { - // No element was found, work on the emptiness check. - if (emptiness_check_right[tmp_buffer_index] != tmp_right) - { - empty = false; - emptiness_check_right[tmp_buffer_index] = - tmp_right; - } - Item *tmp_left = left_[tmp_buffer_index]->load(); - if (emptiness_check_left[tmp_buffer_index] != tmp_left) - { - empty = false; - emptiness_check_left[tmp_buffer_index] = - tmp_left; - } - } - } - - *element = (T)NULL; - return !empty; + uint64_t thread_id = scal::ThreadContext::get().thread_id(); + Item* *emptiness_check_left = + emptiness_check_left_[thread_id]; + Item* *emptiness_check_right = + emptiness_check_right_[thread_id]; + bool empty = true; + // Initialize the result pointer to NULL, which means that no + // element has been removed. + Item *result = NULL; + // Indicates the index which contains the youngest item. + uint64_t buffer_index = -1; + // Memory on the stack frame where timestamps of items can be stored + // temporarily. + uint64_t tmp_timestamp[2][2]; + // Index in the tmp_timestamp array whihc is not used at the moment. + uint64_t tmp_index = 1; + timestamping_->init_sentinel(tmp_timestamp[0]); + uint64_t *timestamp = tmp_timestamp[0]; + // Stores the value of the remove pointer of a thead-local buffer + // before the buffer is actually accessed. + Item* old_right = NULL; + + // Read the start time of the iteration. Items which were timestamped + // after the start time and inserted at the left are not removed. + uint64_t start_time[2]; + timestamping_->read_time(start_time); + // We start iterating over the thread-local lists at a random index. + uint64_t start = hwrand(); + // We iterate over all thead-local buffers + for (uint64_t i = 0; i < num_threads_; i++) { + + uint64_t tmp_buffer_index = (start + i) % num_threads_; + // We get the remove/insert pointer of the current thread-local buffer. + Item* tmp_right = right_[tmp_buffer_index]->load(); + // We get the youngest element from that thread-local buffer. + Item* item = get_right_item(tmp_buffer_index); + // If we found an element, we compare it to the youngest element + // we have found until now. + if (item != NULL) { + empty = false; + uint64_t *item_timestamp; + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + + if (inserted_right(item) && !timestamping_->is_later(invocation_time, item_timestamp)) { + uint64_t expected = 0; + if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + right_[tmp_buffer_index]->compare_exchange_weak( + tmp_right, (Item*)add_next_aba(item, tmp_right, 0)); + *element = item->data.load(); + return true; + } else { + item = get_right_item(tmp_buffer_index); + if (item != NULL) { + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + } + } + } + + if (item != NULL && (result == NULL || is_more_right(item, item_timestamp, result, timestamp))) { + // We found a new youngest element, so we remember it. + result = item; + buffer_index = tmp_buffer_index; + timestamp = item_timestamp; + tmp_index ^=1; + old_right = tmp_right; + } + } else { + // No element was found, work on the emptiness check. + if (emptiness_check_right[tmp_buffer_index] + != tmp_right) { + empty = false; + emptiness_check_right[tmp_buffer_index] = + tmp_right; + } + Item* tmp_left = left_[tmp_buffer_index]->load(); + if (emptiness_check_left[tmp_buffer_index] + != tmp_left) { + empty = false; + emptiness_check_left[tmp_buffer_index] = + tmp_left; + } + } + } + if (result != NULL) { + if (!timestamping_->is_later(timestamp, start_time)) { + // The found item was timestamped after the start of the iteration, + // so it is save to remove it. + uint64_t expected = 0; + if (result->taken.load() == 0) { + if (result->taken.compare_exchange_weak( + expected, 1)) { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + right_[buffer_index]->compare_exchange_weak( + old_right, (Item*)add_next_aba(result, old_right, 0)); + *element = result->data.load(); + return true; + } + } + } + } + + *element = (T)NULL; + return !empty; } }; From 06542ae9ae347d70c7baa05bb1d3150e92687574 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Tue, 18 Dec 2018 23:30:17 +0300 Subject: [PATCH 15/29] fixes insert methods --- cds/container/ts_deque_buffer.h | 236 ++++++++++++++++++++------------ 1 file changed, 149 insertions(+), 87 deletions(-) diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index 5f3e6a7a8..e3161cb05 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -60,17 +60,74 @@ class TSDequeBuffer return (void *)((result & 0xffffffffffffff8) | aba); } - // Returns the leftmost not-taken item from the thread-local list - // indicated by thread_id. - Item *get_left_item(uint64_t thread_id) - { - } - - // Returns the rightmost not-taken item from the thread-local list - // indicated by thread_id. - Item *get_right_item(uint64_t thread_id) - { - } + // Returns the leftmost not-taken item from the thread-local list + // indicated by thread_id. + Item* get_left_item(uint64_t thread_id) { + + // Read the item pointed to by the right pointer. The iteration through + // the linked list can stop at that item. + Item* old_right = right_[thread_id]->load(); + Item* right = (Item*)get_aba_free_pointer(old_right); + int64_t threshold = right->index.load(); + + // Read the leftmost item. + Item* result = (Item*)get_aba_free_pointer(left_[thread_id]->load()); + + // We start at the left pointer and iterate to the right until we + // find the first item which has not been taken yet. + while (true) { + // We reached a node further right than the original right-most + // node. We do not have to search any further to the right, we + // will not take the element anyways. + if (result->index.load() > threshold) { + return NULL; + } + // We found a good node, return it. + if (result->taken.load() == 0) { + return result; + } + // We have reached the end of the list and found nothing, so we + // return NULL. + if (result->right.load() == result) { + return NULL; + } + result = result->right.load(); + } + } + + // Returns the rightmost not-taken item from the thread-local list + // indicated by thread_id. + Item* get_right_item(uint64_t thread_id) { + + // Read the item pointed to by the left pointer. The iteration through + // the linked list can stop at that item. + Item* old_left = left_[thread_id]->load(); + Item* left = (Item*)get_aba_free_pointer(old_left); + int64_t threshold = left->index.load(); + + Item* result = (Item*)get_aba_free_pointer(right_[thread_id]->load()); + + // We start at the right pointer and iterate to the left until we + // find the first item which has not been taken yet. + while (true) { + // We reached a node further left than the original left-most + // node. We do not have to search any further to the left, we + // will not take the element anyways. + if (result->index.load() < threshold) { + return NULL; + } + // We found a good node, return it. + if (result->taken.load() == 0) { + return result; + } + // We have reached the end of the list and found nothing, so we + // return NULL. + if (result->left.load() == result) { + return NULL; + } + result = result->left.load(); + } + } public: void initialize(uint64_t num_threads, TimeStamp *timestamping) @@ -116,89 +173,94 @@ class TSDequeBuffer } } - inline std::atomic *insert_left(T element) - { - uint64_t thread_id = ThreadContext::get().thread_id(); - - // Create a new item. - Item *new_item = new Item(); - timestamping_->init_top_atomic(new_item->timestamp); - new_item->data.store(element); - new_item->taken.store(0); - new_item->left.store(new_item); - // Items inserted at the left get negative indices. Thereby the - // order of items in the thread-local lists correspond with the - // order of indices, and we can use the sign of the index to - // determine on which side an item has been inserted. - new_item->index = -((*next_index_[thread_id])++); - - // Determine leftmost not-taken item in the list. The new item is - // inserted to the left of that item. - Item *old_left = left_[thread_id]->load(); - - Item *left = old_left; - while (left->right.load() != left && left->taken.load()) - { - left = left->right.load(); - } + inline std::atomic *insert_left(T element) { + uint64_t thread_id = scal::ThreadContext::get().thread_id(); - if (left->taken.load() && left->right.load() == left) - { - left = old_left; - left->right.store(left); - Item *old_right = right_[thread_id]->load(); - right_[thread_id]->store(left); - } + // Create a new item. + Item *new_item = new Item(); + timestamping_->init_top_atomic(new_item->timestamp); + new_item->data.store(element); + new_item->taken.store(0); + new_item->left.store(new_item); + // Items inserted at the left get negative indices. Thereby the + // order of items in the thread-local lists correspond with the + // order of indices, and we can use the sign of the index to + // determine on which side an item has been inserted. + new_item->index = -((*next_index_[thread_id])++); + + // Determine leftmost not-taken item in the list. The new item is + // inserted to the left of that item. + Item* old_left = left_[thread_id]->load(); + + Item* left = (Item*)get_aba_free_pointer(old_left); + while (left->right.load() != left + && left->taken.load()) { + left = left->right.load(); + } - // Add the new item to the list. - new_item->right.store(left); - left->left.store(new_item); - left_[thread_id]->store(new_item); + if (left->taken.load() && left->right.load() == left) { + // The buffer is empty. We have to increase the aba counter of the + // right pointer too to guarantee that a pending right-pointer + // update of a remove operation does not make the left and the + // right pointer point to different lists. - // Return a pointer to the timestamp location of the item so that a - // timestamp can be added. - return new_item->timestamp; - } + left = (Item*)get_aba_free_pointer(old_left); + left->right.store(left); + Item* old_right = right_[thread_id]->load(); + right_[thread_id]->store((Item*) add_next_aba(left, old_right, 1)); + } - inline std::atomic *insert_right(T element) - { - uint64_t thread_id = ThreadContext::get().thread_id(); - - // Create a new item. - Item *new_item = new Item(); - timestamping_->init_top_atomic(new_item->timestamp); - new_item->data.store(element); - new_item->taken.store(0); - new_item->right.store(new_item); - new_item->index = (*next_index_[thread_id])++; - - // Determine the rightmost not-taken item in the list. The new item is - // inserted to the right of that item. - Item *old_right = right_[thread_id]->load(); - - Item *right = old_right; - while (right->left.load() != right && right->taken.load()) - { - right = right->left.load(); - } + // Add the new item to the list. + new_item->right.store(left); + left->left.store(new_item); + left_[thread_id]->store( + (Item*) add_next_aba(new_item, old_left, 1)); - if (right->taken.load() && right->left.load() == right) - { - right = old_right; - right->left.store(right); - Item *old_left = left_[thread_id]->load(); - left_[thread_id]->store(right); - } + // Return a pointer to the timestamp location of the item so that a + // timestamp can be added. + return new_item->timestamp; + } - // Add the new item to the list. - new_item->left.store(right); - right->right.store(new_item); - right_[thread_id]->store(new_item); + inline std::atomic *insert_right(T element) { + uint64_t thread_id = scal::ThreadContext::get().thread_id(); - // Return a pointer to the timestamp location of the item so that a - // timestamp can be added. - return new_item->timestamp; - } + // Create a new item. + Item *new_item = new Item(); + timestamping_->init_top_atomic(new_item->timestamp); + new_item->data.store(element); + new_item->taken.store(0); + new_item->right.store(new_item); + new_item->index = (*next_index_[thread_id])++; + + // Determine the rightmost not-taken item in the list. The new item is + // inserted to the right of that item. + Item* old_right = right_[thread_id]->load(); + + Item* right = (Item*)get_aba_free_pointer(old_right); + while (right->left.load() != right + && right->taken.load()) { + right = right->left.load(); + } + + if (right->taken.load() && right->left.load() == right) { + // The buffer is empty. We have to increase the aba counter of the + // left pointer too to guarantee that a pending left-pointer + // update of a remove operation does not make the left and the + // right pointer point to different lists. + right = (Item*)get_aba_free_pointer(old_right); + right->left.store(right); + Item* old_left = left_[thread_id]->load(); + left_[thread_id]->store( (Item*) add_next_aba(right, old_left, 1)); } + + // Add the new item to the list. + new_item->left.store(right); + right->right.store(new_item); + right_[thread_id]->store((Item*) add_next_aba(new_item, old_right, 1)); + + // Return a pointer to the timestamp location of the item so that a + // timestamp can be added. + return new_item->timestamp; + } // Helper function which returns true if the item was inserted at the left. inline bool inserted_left(Item *item) From 09052d70d34278799fabe3495bbd4db9978de038 Mon Sep 17 00:00:00 2001 From: vikkiorrikki Date: Wed, 19 Dec 2018 20:17:55 +0300 Subject: [PATCH 16/29] timestamp atomic counter --- cds/container/ts_timestamp.h | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/cds/container/ts_timestamp.h b/cds/container/ts_timestamp.h index 0ea907a80..2d0291ded 100644 --- a/cds/container/ts_timestamp.h +++ b/cds/container/ts_timestamp.h @@ -1,3 +1,5 @@ +#include + inline uint64_t get_hwptime(void) { uint64_t aux; @@ -99,4 +101,51 @@ class HardwareIntervalTimestamp { inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) { return timestamp2[1] < timestamp1[0]; } +}; + +class AtomicCounterTimestamp { + private: + std::atomic *clock_; + + public: + inline void initialize(uint64_t delay, uint64_t num_threads) { + clock_ = new std::atomic(); + clock_->store(1); + } + + inline void init_sentinel(uint64_t *result) { + result[0] = 0; + } + + inline void init_sentinel_atomic(std::atomic *result) { + result[0].store(0); + } + + inline void init_top_atomic(std::atomic *result) { + result[0].store(UINT64_MAX); + } + + inline void init_top(uint64_t *result) { + result[0] = UINT64_MAX; + } + + inline void load_timestamp(uint64_t *result, std::atomic *source) { + result[0] = source[0].load(); + } + + inline void set_timestamp(std::atomic *result) { + result[0].store(clock_->fetch_add(1)); + } + + inline void set_timestamp_local(uint64_t *result) { + result[0] = clock_->fetch_add(1); + } + + inline void read_time(uint64_t *result) { + result[0] = clock_->load(); + } + + inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) { + return timestamp2[0] < timestamp1[0]; + } }; \ No newline at end of file From 546b8b3084818a581f4a96d6378cf9e015126973 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Wed, 19 Dec 2018 23:06:17 +0300 Subject: [PATCH 17/29] assign threadcontext to each thread --- cds/container/threadcontext.h | 14 +++++++++++++- cds/container/ts_deque.h | 7 +++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/cds/container/threadcontext.h b/cds/container/threadcontext.h index 4a4f10f7b..ca7f5e163 100644 --- a/cds/container/threadcontext.h +++ b/cds/container/threadcontext.h @@ -5,6 +5,7 @@ class ThreadContext { public: static ThreadContext& get(); static void prepare(uint64_t num_threads); + static void assign_context(); inline uint64_t thread_id() { return thread_id_; @@ -26,11 +27,22 @@ pthread_key_t ThreadContext::threadcontext_key; ThreadContext *ThreadContext::contexts[kMaxThreads]; ThreadContext& ThreadContext::get() { + if (pthread_getspecific(threadcontext_key) == NULL) { + assign_context(); + } ThreadContext *context = static_cast( pthread_getspecific(threadcontext_key)); return *context; } +void ThreadContext::assign_context() { + uint64_t thread_id = __sync_fetch_and_add(&global_thread_id_cnt, 1); + if (pthread_setspecific(threadcontext_key, contexts[thread_id])) { + fprintf(stderr, "%s: pthread_setspecific failed\n", __func__); + exit(EXIT_FAILURE); + } +} + void ThreadContext::prepare(uint64_t num_threads) { pthread_key_create(&threadcontext_key, NULL); for (uint64_t i = 0; i < num_threads; i++) { @@ -38,4 +50,4 @@ void ThreadContext::prepare(uint64_t num_threads) { context->thread_id_ = i; contexts[i] = context; } -} \ No newline at end of file +} diff --git a/cds/container/ts_deque.h b/cds/container/ts_deque.h index 6aa9f82fc..021682b5a 100644 --- a/cds/container/ts_deque.h +++ b/cds/container/ts_deque.h @@ -1,18 +1,21 @@ #include #include "ts_deque_buffer.h" +#include "threadcontext.h" template class TSDeque { private: - TSDequeBuffer *buffer_; Timestamp *timestamping_; public: - TSDeque (uint64_t num_threads) + TSDeque (uint64_t num_threads, uint64_t delay) { + ThreadContext::prepare(num_threads); + ThreadContext::assign_context(); timestamping_ = new Timestamp(); + timestamping_->initialize(delay, num_threads); buffer_ = new TSDequeBuffer(); buffer_->initialize(num_threads, timestamping_); From 0e4fd300b18cbbbd7b09b64c4f120a0aa6ad2767 Mon Sep 17 00:00:00 2001 From: EduardBlees Date: Fri, 21 Dec 2018 01:13:37 +0300 Subject: [PATCH 18/29] code format --- cds/container/threadcontext.h | 88 +-- cds/container/ts_deque.h | 91 +-- cds/container/ts_deque_buffer.h | 1066 ++++++++++++++++--------------- cds/container/ts_timestamp.h | 192 +++--- 4 files changed, 766 insertions(+), 671 deletions(-) diff --git a/cds/container/threadcontext.h b/cds/container/threadcontext.h index ca7f5e163..046c4b3f4 100644 --- a/cds/container/threadcontext.h +++ b/cds/container/threadcontext.h @@ -1,53 +1,61 @@ #include #include -class ThreadContext { - public: - static ThreadContext& get(); - static void prepare(uint64_t num_threads); - static void assign_context(); - - inline uint64_t thread_id() { - return thread_id_; - } - - private: - static constexpr uint64_t kMaxThreads = 1024; - static uint64_t global_thread_id_cnt; - static ThreadContext *contexts[kMaxThreads]; - static pthread_key_t threadcontext_key; - - ThreadContext() {} - - uint64_t thread_id_; +class ThreadContext +{ +public: + static ThreadContext &get(); + static void prepare(uint64_t num_threads); + static void assign_context(); + + inline uint64_t thread_id() + { + return thread_id_; + } + +private: + static constexpr uint64_t kMaxThreads = 1024; + static uint64_t global_thread_id_cnt; + static ThreadContext *contexts[kMaxThreads]; + static pthread_key_t threadcontext_key; + + ThreadContext() {} + + uint64_t thread_id_; }; uint64_t ThreadContext::global_thread_id_cnt = 0; pthread_key_t ThreadContext::threadcontext_key; ThreadContext *ThreadContext::contexts[kMaxThreads]; -ThreadContext& ThreadContext::get() { - if (pthread_getspecific(threadcontext_key) == NULL) { - assign_context(); - } - ThreadContext *context = static_cast( - pthread_getspecific(threadcontext_key)); - return *context; +ThreadContext &ThreadContext::get() +{ + if (pthread_getspecific(threadcontext_key) == NULL) + { + assign_context(); + } + ThreadContext *context = static_cast( + pthread_getspecific(threadcontext_key)); + return *context; } -void ThreadContext::assign_context() { - uint64_t thread_id = __sync_fetch_and_add(&global_thread_id_cnt, 1); - if (pthread_setspecific(threadcontext_key, contexts[thread_id])) { - fprintf(stderr, "%s: pthread_setspecific failed\n", __func__); - exit(EXIT_FAILURE); - } +void ThreadContext::assign_context() +{ + uint64_t thread_id = __sync_fetch_and_add(&global_thread_id_cnt, 1); + if (pthread_setspecific(threadcontext_key, contexts[thread_id])) + { + fprintf(stderr, "%s: pthread_setspecific failed\n", __func__); + exit(EXIT_FAILURE); + } } -void ThreadContext::prepare(uint64_t num_threads) { - pthread_key_create(&threadcontext_key, NULL); - for (uint64_t i = 0; i < num_threads; i++) { - ThreadContext *context = new ThreadContext(); - context->thread_id_ = i; - contexts[i] = context; - } -} +void ThreadContext::prepare(uint64_t num_threads) +{ + pthread_key_create(&threadcontext_key, NULL); + for (uint64_t i = 0; i < num_threads; i++) + { + ThreadContext *context = new ThreadContext(); + context->thread_id_ = i; + contexts[i] = context; + } +} \ No newline at end of file diff --git a/cds/container/ts_deque.h b/cds/container/ts_deque.h index 021682b5a..fa027f5c2 100644 --- a/cds/container/ts_deque.h +++ b/cds/container/ts_deque.h @@ -2,74 +2,79 @@ #include "ts_deque_buffer.h" #include "threadcontext.h" -template -class TSDeque { - private: +template +class TSDeque +{ +private: TSDequeBuffer *buffer_; Timestamp *timestamping_; - public: - TSDeque (uint64_t num_threads, uint64_t delay) +public: + TSDeque(uint64_t num_threads, uint64_t delay) { - ThreadContext::prepare(num_threads); - ThreadContext::assign_context(); + ThreadContext::prepare(num_threads); + ThreadContext::assign_context(); - timestamping_ = new Timestamp(); - timestamping_->initialize(delay, num_threads); + timestamping_ = new Timestamp(); + timestamping_->initialize(delay, num_threads); - buffer_ = new TSDequeBuffer(); - buffer_->initialize(num_threads, timestamping_); + buffer_ = new TSDequeBuffer(); + buffer_->initialize(num_threads, timestamping_); } bool insert_left(T element) { - std::atomic *item = buffer_->insert_left(element); - // In the set_timestamp operation first a new timestamp is acquired - // and then assigned to the item. The operation may not be executed - // atomically. - timestamping_->set_timestamp(item); - return true; + std::atomic *item = buffer_->insert_left(element); + // In the set_timestamp operation first a new timestamp is acquired + // and then assigned to the item. The operation may not be executed + // atomically. + timestamping_->set_timestamp(item); + return true; } bool insert_right(T element) { - std::atomic *item = buffer_->insert_right(element); - // In the set_timestamp operation first a new timestamp is acquired - // and then assigned to the item. The operation may not be executed - // atomically. - timestamping_->set_timestamp(item); - return true; + std::atomic *item = buffer_->insert_right(element); + // In the set_timestamp operation first a new timestamp is acquired + // and then assigned to the item. The operation may not be executed + // atomically. + timestamping_->set_timestamp(item); + return true; } bool remove_left(T *element) { - // Read the invocation time of this operation, needed for the - // elimination optimization. - uint64_t invocation_time[2]; - timestamping_->read_time(invocation_time); - while (buffer_->try_remove_left(element, invocation_time)) { + // Read the invocation time of this operation, needed for the + // elimination optimization. + uint64_t invocation_time[2]; + timestamping_->read_time(invocation_time); + while (buffer_->try_remove_left(element, invocation_time)) + { - if (*element != (T)NULL) { - return true; + if (*element != (T)NULL) + { + return true; + } } - } - // The deque was empty, return false. - return false; + // The deque was empty, return false. + return false; } bool remove_right(T *element) { - // Read the invocation time of this operation, needed for the - // elimination optimization. - uint64_t invocation_time[2]; - timestamping_->read_time(invocation_time); - while (buffer_->try_remove_right(element, invocation_time)) { + // Read the invocation time of this operation, needed for the + // elimination optimization. + uint64_t invocation_time[2]; + timestamping_->read_time(invocation_time); + while (buffer_->try_remove_right(element, invocation_time)) + { - if (*element != (T)NULL) { - return true; + if (*element != (T)NULL) + { + return true; + } } - } - // The deque was empty, return false. - return false; + // The deque was empty, return false. + return false; } }; diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index e3161cb05..bee83fc6d 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -3,571 +3,619 @@ #include #include "threadcontext.h" -inline uint64_t rdtsc() { - unsigned int hi, lo; - __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); - return ((uint64_t) lo) | (((uint64_t) hi) << 32); +inline uint64_t rdtsc() +{ + unsigned int hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t)lo) | (((uint64_t)hi) << 32); } -inline uint64_t hwrand() { - return (rdtsc() >> 6); +inline uint64_t hwrand() +{ + return (rdtsc() >> 6); } template class TSDequeBuffer { - private: - typedef struct Item - { - std::atomic left; - std::atomic right; - std::atomic taken; - std::atomic data; - std::atomic timestamp[2]; - // Insertion index, needed for the termination condition in - // get_left_item. Items inserted at the left get negative - // indices, items inserted at the right get positive indices. - std::atomic index; - } Item; - - // The number of threads. - uint64_t num_threads_; - std::atomic **left_; - std::atomic **right_; - int64_t **next_index_; - // The pointers for the emptiness check. - Item ***emptiness_check_left_; - Item ***emptiness_check_right_; - TimeStamp *timestamping_; - - // Helper function to remove the ABA counter from a pointer. - void *get_aba_free_pointer(void *pointer) - { - uint64_t result = (uint64_t)pointer; - result &= 0xfffffffffffffff8; - return (void *)result; - } - - // Helper function which retrieves the ABA counter of a pointer old - // and sets this ABA counter + increment to the pointer pointer. - void *add_next_aba(void *pointer, void *old, uint64_t increment) - { - uint64_t aba = (uint64_t)old; - aba += increment; - aba &= 0x7; - uint64_t result = (uint64_t)pointer; - result = (result & 0xfffffffffffffff8) | aba; - return (void *)((result & 0xffffffffffffff8) | aba); - } +private: + typedef struct Item + { + std::atomic left; + std::atomic right; + std::atomic taken; + std::atomic data; + std::atomic timestamp[2]; + // Insertion index, needed for the termination condition in + // get_left_item. Items inserted at the left get negative + // indices, items inserted at the right get positive indices. + std::atomic index; + } Item; + + // The number of threads. + uint64_t num_threads_; + std::atomic **left_; + std::atomic **right_; + int64_t **next_index_; + // The pointers for the emptiness check. + Item ***emptiness_check_left_; + Item ***emptiness_check_right_; + TimeStamp *timestamping_; + + // Helper function to remove the ABA counter from a pointer. + void *get_aba_free_pointer(void *pointer) + { + uint64_t result = (uint64_t)pointer; + result &= 0xfffffffffffffff8; + return (void *)result; + } + + // Helper function which retrieves the ABA counter of a pointer old + // and sets this ABA counter + increment to the pointer pointer. + void *add_next_aba(void *pointer, void *old, uint64_t increment) + { + uint64_t aba = (uint64_t)old; + aba += increment; + aba &= 0x7; + uint64_t result = (uint64_t)pointer; + result = (result & 0xfffffffffffffff8) | aba; + return (void *)((result & 0xffffffffffffff8) | aba); + } // Returns the leftmost not-taken item from the thread-local list // indicated by thread_id. - Item* get_left_item(uint64_t thread_id) { - - // Read the item pointed to by the right pointer. The iteration through - // the linked list can stop at that item. - Item* old_right = right_[thread_id]->load(); - Item* right = (Item*)get_aba_free_pointer(old_right); - int64_t threshold = right->index.load(); - - // Read the leftmost item. - Item* result = (Item*)get_aba_free_pointer(left_[thread_id]->load()); - - // We start at the left pointer and iterate to the right until we - // find the first item which has not been taken yet. - while (true) { - // We reached a node further right than the original right-most - // node. We do not have to search any further to the right, we - // will not take the element anyways. - if (result->index.load() > threshold) { - return NULL; - } - // We found a good node, return it. - if (result->taken.load() == 0) { - return result; - } - // We have reached the end of the list and found nothing, so we - // return NULL. - if (result->right.load() == result) { - return NULL; + Item *get_left_item(uint64_t thread_id) + { + + // Read the item pointed to by the right pointer. The iteration through + // the linked list can stop at that item. + Item *old_right = right_[thread_id]->load(); + Item *right = (Item *)get_aba_free_pointer(old_right); + int64_t threshold = right->index.load(); + + // Read the leftmost item. + Item *result = (Item *)get_aba_free_pointer(left_[thread_id]->load()); + + // We start at the left pointer and iterate to the right until we + // find the first item which has not been taken yet. + while (true) + { + // We reached a node further right than the original right-most + // node. We do not have to search any further to the right, we + // will not take the element anyways. + if (result->index.load() > threshold) + { + return NULL; + } + // We found a good node, return it. + if (result->taken.load() == 0) + { + return result; + } + // We have reached the end of the list and found nothing, so we + // return NULL. + if (result->right.load() == result) + { + return NULL; + } + result = result->right.load(); } - result = result->right.load(); - } } // Returns the rightmost not-taken item from the thread-local list // indicated by thread_id. - Item* get_right_item(uint64_t thread_id) { - - // Read the item pointed to by the left pointer. The iteration through - // the linked list can stop at that item. - Item* old_left = left_[thread_id]->load(); - Item* left = (Item*)get_aba_free_pointer(old_left); - int64_t threshold = left->index.load(); - - Item* result = (Item*)get_aba_free_pointer(right_[thread_id]->load()); - - // We start at the right pointer and iterate to the left until we - // find the first item which has not been taken yet. - while (true) { - // We reached a node further left than the original left-most - // node. We do not have to search any further to the left, we - // will not take the element anyways. - if (result->index.load() < threshold) { - return NULL; - } - // We found a good node, return it. - if (result->taken.load() == 0) { - return result; - } - // We have reached the end of the list and found nothing, so we - // return NULL. - if (result->left.load() == result) { - return NULL; + Item *get_right_item(uint64_t thread_id) + { + + // Read the item pointed to by the left pointer. The iteration through + // the linked list can stop at that item. + Item *old_left = left_[thread_id]->load(); + Item *left = (Item *)get_aba_free_pointer(old_left); + int64_t threshold = left->index.load(); + + Item *result = (Item *)get_aba_free_pointer(right_[thread_id]->load()); + + // We start at the right pointer and iterate to the left until we + // find the first item which has not been taken yet. + while (true) + { + // We reached a node further left than the original left-most + // node. We do not have to search any further to the left, we + // will not take the element anyways. + if (result->index.load() < threshold) + { + return NULL; + } + // We found a good node, return it. + if (result->taken.load() == 0) + { + return result; + } + // We have reached the end of the list and found nothing, so we + // return NULL. + if (result->left.load() == result) + { + return NULL; + } + result = result->left.load(); } - result = result->left.load(); - } } - public: - void initialize(uint64_t num_threads, TimeStamp *timestamping) - { +public: + void initialize(uint64_t num_threads, TimeStamp *timestamping) + { - num_threads_ = num_threads; - timestamping_ = timestamping; + num_threads_ = num_threads; + timestamping_ = timestamping; - left_ = new std::atomic *[num_threads_]; + left_ = new std::atomic *[num_threads_]; - right_ = new std::atomic *[num_threads_]; + right_ = new std::atomic *[num_threads_]; - next_index_ = new int64_t *[num_threads_]; + next_index_ = new int64_t *[num_threads_]; - emptiness_check_left_ = new Item **[num_threads_]; + emptiness_check_left_ = new Item **[num_threads_]; - emptiness_check_right_ = new Item **[num_threads_]; + emptiness_check_right_ = new Item **[num_threads_]; - for (uint64_t i = 0; i < num_threads_; i++) - { + for (uint64_t i = 0; i < num_threads_; i++) + { - left_[i] = new std::atomic(); + left_[i] = new std::atomic(); - right_[i] = new std::atomic(); + right_[i] = new std::atomic(); - next_index_[i] = new int64_t(); + next_index_[i] = new int64_t(); - // Add a sentinal node. - Item *new_item = new Item(); - timestamping_->init_sentinel_atomic(new_item->timestamp); - new_item->data.store(0); - new_item->taken.store(1); - new_item->left.store(new_item); - new_item->right.store(new_item); - new_item->index.store(0); - left_[i]->store(new_item); - right_[i]->store(new_item); - *next_index_[i] = 1; + // Add a sentinal node. + Item *new_item = new Item(); + timestamping_->init_sentinel_atomic(new_item->timestamp); + new_item->data.store(0); + new_item->taken.store(1); + new_item->left.store(new_item); + new_item->right.store(new_item); + new_item->index.store(0); + left_[i]->store(new_item); + right_[i]->store(new_item); + *next_index_[i] = 1; - emptiness_check_left_[i] = new Item *[num_threads_]; + emptiness_check_left_[i] = new Item *[num_threads_]; - emptiness_check_right_[i] = new Item *[num_threads_]; - } - } + emptiness_check_right_[i] = new Item *[num_threads_]; + } + } + + inline std::atomic *insert_left(T element) + { + uint64_t thread_id = scal::ThreadContext::get().thread_id(); + + // Create a new item. + Item *new_item = new Item(); + timestamping_->init_top_atomic(new_item->timestamp); + new_item->data.store(element); + new_item->taken.store(0); + new_item->left.store(new_item); + // Items inserted at the left get negative indices. Thereby the + // order of items in the thread-local lists correspond with the + // order of indices, and we can use the sign of the index to + // determine on which side an item has been inserted. + new_item->index = -((*next_index_[thread_id])++); + + // Determine leftmost not-taken item in the list. The new item is + // inserted to the left of that item. + Item *old_left = left_[thread_id]->load(); + + Item *left = (Item *)get_aba_free_pointer(old_left); + while (left->right.load() != left && left->taken.load()) + { + left = left->right.load(); + } + + if (left->taken.load() && left->right.load() == left) + { + // The buffer is empty. We have to increase the aba counter of the + // right pointer too to guarantee that a pending right-pointer + // update of a remove operation does not make the left and the + // right pointer point to different lists. + + left = (Item *)get_aba_free_pointer(old_left); + left->right.store(left); + Item *old_right = right_[thread_id]->load(); + right_[thread_id]->store((Item *)add_next_aba(left, old_right, 1)); + } + + // Add the new item to the list. + new_item->right.store(left); + left->left.store(new_item); + left_[thread_id]->store( + (Item *)add_next_aba(new_item, old_left, 1)); - inline std::atomic *insert_left(T element) { - uint64_t thread_id = scal::ThreadContext::get().thread_id(); + // Return a pointer to the timestamp location of the item so that a + // timestamp can be added. + return new_item->timestamp; + } - // Create a new item. - Item *new_item = new Item(); - timestamping_->init_top_atomic(new_item->timestamp); - new_item->data.store(element); - new_item->taken.store(0); - new_item->left.store(new_item); - // Items inserted at the left get negative indices. Thereby the - // order of items in the thread-local lists correspond with the - // order of indices, and we can use the sign of the index to - // determine on which side an item has been inserted. - new_item->index = -((*next_index_[thread_id])++); + inline std::atomic *insert_right(T element) + { + uint64_t thread_id = scal::ThreadContext::get().thread_id(); + + // Create a new item. + Item *new_item = new Item(); + timestamping_->init_top_atomic(new_item->timestamp); + new_item->data.store(element); + new_item->taken.store(0); + new_item->right.store(new_item); + new_item->index = (*next_index_[thread_id])++; + + // Determine the rightmost not-taken item in the list. The new item is + // inserted to the right of that item. + Item *old_right = right_[thread_id]->load(); + + Item *right = (Item *)get_aba_free_pointer(old_right); + while (right->left.load() != right && right->taken.load()) + { + right = right->left.load(); + } - // Determine leftmost not-taken item in the list. The new item is - // inserted to the left of that item. - Item* old_left = left_[thread_id]->load(); + if (right->taken.load() && right->left.load() == right) + { + // The buffer is empty. We have to increase the aba counter of the + // left pointer too to guarantee that a pending left-pointer + // update of a remove operation does not make the left and the + // right pointer point to different lists. + right = (Item *)get_aba_free_pointer(old_right); + right->left.store(right); + Item *old_left = left_[thread_id]->load(); + left_[thread_id]->store((Item *)add_next_aba(right, old_left, 1)); + } - Item* left = (Item*)get_aba_free_pointer(old_left); - while (left->right.load() != left - && left->taken.load()) { - left = left->right.load(); - } + // Add the new item to the list. + new_item->left.store(right); + right->right.store(new_item); + right_[thread_id]->store((Item *)add_next_aba(new_item, old_right, 1)); - if (left->taken.load() && left->right.load() == left) { - // The buffer is empty. We have to increase the aba counter of the - // right pointer too to guarantee that a pending right-pointer - // update of a remove operation does not make the left and the - // right pointer point to different lists. + // Return a pointer to the timestamp location of the item so that a + // timestamp can be added. + return new_item->timestamp; + } - left = (Item*)get_aba_free_pointer(old_left); - left->right.store(left); - Item* old_right = right_[thread_id]->load(); - right_[thread_id]->store((Item*) add_next_aba(left, old_right, 1)); - } + // Helper function which returns true if the item was inserted at the left. + inline bool inserted_left(Item *item) + { + return item->index.load() < 0; + } - // Add the new item to the list. - new_item->right.store(left); - left->left.store(new_item); - left_[thread_id]->store( - (Item*) add_next_aba(new_item, old_left, 1)); + // Helper function which returns true if the item was inserted at the right. + inline bool inserted_right(Item *item) + { + return item->index.load() > 0; + } - // Return a pointer to the timestamp location of the item so that a - // timestamp can be added. - return new_item->timestamp; + // Helper function which returns true if item1 is more left than item2. + inline bool is_more_left(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) + { + if (inserted_left(item2)) + { + if (inserted_left(item1)) + { + return timestamping_->is_later(timestamp1, timestamp2); + } + else + { + return false; + } + } + else + { + if (inserted_left(item1)) + { + return true; + } + else + { + return timestamping_->is_later(timestamp2, timestamp1); + } + } } - inline std::atomic *insert_right(T element) { - uint64_t thread_id = scal::ThreadContext::get().thread_id(); - - // Create a new item. - Item *new_item = new Item(); - timestamping_->init_top_atomic(new_item->timestamp); - new_item->data.store(element); - new_item->taken.store(0); - new_item->right.store(new_item); - new_item->index = (*next_index_[thread_id])++; - - // Determine the rightmost not-taken item in the list. The new item is - // inserted to the right of that item. - Item* old_right = right_[thread_id]->load(); - - Item* right = (Item*)get_aba_free_pointer(old_right); - while (right->left.load() != right - && right->taken.load()) { - right = right->left.load(); - } - - if (right->taken.load() && right->left.load() == right) { - // The buffer is empty. We have to increase the aba counter of the - // left pointer too to guarantee that a pending left-pointer - // update of a remove operation does not make the left and the - // right pointer point to different lists. - right = (Item*)get_aba_free_pointer(old_right); - right->left.store(right); - Item* old_left = left_[thread_id]->load(); - left_[thread_id]->store( (Item*) add_next_aba(right, old_left, 1)); } - - // Add the new item to the list. - new_item->left.store(right); - right->right.store(new_item); - right_[thread_id]->store((Item*) add_next_aba(new_item, old_right, 1)); - - // Return a pointer to the timestamp location of the item so that a - // timestamp can be added. - return new_item->timestamp; + // Helper function which returns true if item1 is more right than item2. + inline bool is_more_right(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) + { + if (inserted_right(item2)) + { + if (inserted_right(item1)) + { + return timestamping_->is_later(timestamp1, timestamp2); + } + else + { + return false; + } + } + else + { + if (inserted_right(item1)) + { + return true; + } + else + { + return timestamping_->is_later(timestamp2, timestamp1); + } + } } - // Helper function which returns true if the item was inserted at the left. - inline bool inserted_left(Item *item) - { - return item->index.load() < 0; - } - - // Helper function which returns true if the item was inserted at the right. - inline bool inserted_right(Item *item) - { - return item->index.load() > 0; - } - - // Helper function which returns true if item1 is more left than item2. - inline bool is_more_left(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) - { - if (inserted_left(item2)) - { - if (inserted_left(item1)) - { - return timestamping_->is_later(timestamp1, timestamp2); - } - else - { - return false; - } - } - else - { - if (inserted_left(item1)) - { - return true; - } - else - { - return timestamping_->is_later(timestamp2, timestamp1); - } - } - } - - // Helper function which returns true if item1 is more right than item2. - inline bool is_more_right(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) - { - if (inserted_right(item2)) - { - if (inserted_right(item1)) - { - return timestamping_->is_later(timestamp1, timestamp2); - } - else - { - return false; - } - } - else - { - if (inserted_right(item1)) - { - return true; - } - else - { - return timestamping_->is_later(timestamp2, timestamp1); - } - } - } - - bool try_remove_left(T *element, uint64_t *invocation_time) - { - // Initialize the data needed for the emptiness check. - uint64_t thread_id = scal::ThreadContext::get().thread_id(); - Item* *emptiness_check_left = - emptiness_check_left_[thread_id]; - Item* *emptiness_check_right = - emptiness_check_right_[thread_id]; - bool empty = true; - // Initialize the result pointer to NULL, which means that no - // element has been removed. - Item *result = NULL; - // Indicates the index which contains the youngest item. - uint64_t buffer_index = -1; - // Memory on the stack frame where timestamps of items can be stored - // temporarily. - uint64_t tmp_timestamp[2][2]; - // Index in the tmp_timestamp array which is not used at the moment. - uint64_t tmp_index = 1; - timestamping_->init_sentinel(tmp_timestamp[0]); - uint64_t *timestamp = tmp_timestamp[0]; - // Stores the value of the remove pointer of a thead-local buffer - // before the buffer is actually accessed. - Item* old_left = NULL; - - // Read the start time of the iteration. Items which were timestamped - // after the start time and inserted at the right are not removed. - uint64_t start_time[2]; - timestamping_->read_time(start_time); - // We start iterating over the thread-local lists at a random index. - uint64_t start = hwrand(); - // We iterate over all thead-local buffers - for (uint64_t i = 0; i < num_threads_; i++) { - - uint64_t tmp_buffer_index = (start + i) % num_threads_; - // We get the remove/insert pointer of the current thread-local buffer. - Item* tmp_left = left_[tmp_buffer_index]->load(); - // We get the youngest element from that thread-local buffer. - Item* item = get_left_item(tmp_buffer_index); - // If we found an element, we compare it to the youngest element - // we have found until now. - if (item != NULL) { - empty = false; - uint64_t *item_timestamp; - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; - - if (inserted_left(item) && !timestamping_->is_later(invocation_time, item_timestamp)) { - uint64_t expected = 0; - if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - left_[tmp_buffer_index]->compare_exchange_weak( - tmp_left, (Item*)add_next_aba(item, tmp_left, 0)); - *element = item->data.load(); - return true; - } else { - item = get_left_item(tmp_buffer_index); - if (item != NULL) { + bool try_remove_left(T *element, uint64_t *invocation_time) + { + // Initialize the data needed for the emptiness check. + uint64_t thread_id = scal::ThreadContext::get().thread_id(); + Item **emptiness_check_left = + emptiness_check_left_[thread_id]; + Item **emptiness_check_right = + emptiness_check_right_[thread_id]; + bool empty = true; + // Initialize the result pointer to NULL, which means that no + // element has been removed. + Item *result = NULL; + // Indicates the index which contains the youngest item. + uint64_t buffer_index = -1; + // Memory on the stack frame where timestamps of items can be stored + // temporarily. + uint64_t tmp_timestamp[2][2]; + // Index in the tmp_timestamp array which is not used at the moment. + uint64_t tmp_index = 1; + timestamping_->init_sentinel(tmp_timestamp[0]); + uint64_t *timestamp = tmp_timestamp[0]; + // Stores the value of the remove pointer of a thead-local buffer + // before the buffer is actually accessed. + Item *old_left = NULL; + + // Read the start time of the iteration. Items which were timestamped + // after the start time and inserted at the right are not removed. + uint64_t start_time[2]; + timestamping_->read_time(start_time); + // We start iterating over the thread-local lists at a random index. + uint64_t start = hwrand(); + // We iterate over all thead-local buffers + for (uint64_t i = 0; i < num_threads_; i++) + { + + uint64_t tmp_buffer_index = (start + i) % num_threads_; + // We get the remove/insert pointer of the current thread-local buffer. + Item *tmp_left = left_[tmp_buffer_index]->load(); + // We get the youngest element from that thread-local buffer. + Item *item = get_left_item(tmp_buffer_index); + // If we found an element, we compare it to the youngest element + // we have found until now. + if (item != NULL) + { + empty = false; + uint64_t *item_timestamp; timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); item_timestamp = tmp_timestamp[tmp_index]; - } + + if (inserted_left(item) && !timestamping_->is_later(invocation_time, item_timestamp)) + { + uint64_t expected = 0; + if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + left_[tmp_buffer_index]->compare_exchange_weak( + tmp_left, (Item *)add_next_aba(item, tmp_left, 0)); + *element = item->data.load(); + return true; + } + else + { + item = get_left_item(tmp_buffer_index); + if (item != NULL) + { + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + } + } + } + + if (item != NULL && (result == NULL || is_more_left(item, item_timestamp, result, timestamp))) + { + // We found a new leftmost item, so we remember it. + result = item; + buffer_index = tmp_buffer_index; + timestamp = item_timestamp; + tmp_index ^= 1; + old_left = tmp_left; + + // Check if we can remove the element immediately. + if (inserted_left(result) && !timestamping_->is_later(invocation_time, timestamp)) + { + uint64_t expected = 0; + if (result->taken.load() == 0) + { + if (result->taken.compare_exchange_weak( + expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + left_[buffer_index]->compare_exchange_weak( + old_left, (Item *)add_next_aba(result, old_left, 0)); + + *element = result->data.load(); + return true; + } + } + } + } } - } - - if (item != NULL && (result == NULL || is_more_left(item, item_timestamp, result, timestamp))) { - // We found a new leftmost item, so we remember it. - result = item; - buffer_index = tmp_buffer_index; - timestamp = item_timestamp; - tmp_index ^=1; - old_left = tmp_left; - - // Check if we can remove the element immediately. - if (inserted_left(result) && !timestamping_->is_later(invocation_time, timestamp)) { - uint64_t expected = 0; - if (result->taken.load() == 0) { - if (result->taken.compare_exchange_weak( - expected, 1)) { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - left_[buffer_index]->compare_exchange_weak( - old_left, (Item*)add_next_aba(result, old_left, 0)); - - *element = result->data.load(); - return true; + else + { + // No element was found, work on the emptiness check. + if (emptiness_check_left[tmp_buffer_index] != tmp_left) + { + empty = false; + emptiness_check_left[tmp_buffer_index] = + tmp_left; + } + Item *tmp_right = right_[tmp_buffer_index]->load(); + if (emptiness_check_right[tmp_buffer_index] != tmp_right) + { + empty = false; + emptiness_check_right[tmp_buffer_index] = + tmp_right; } - } } - } - } else { - // No element was found, work on the emptiness check. - if (emptiness_check_left[tmp_buffer_index] - != tmp_left) { - empty = false; - emptiness_check_left[tmp_buffer_index] = - tmp_left; - } - Item* tmp_right = right_[tmp_buffer_index]->load(); - if (emptiness_check_right[tmp_buffer_index] - != tmp_right) { - empty = false; - emptiness_check_right[tmp_buffer_index] = - tmp_right; - } } - } - if (result != NULL) { - if (!timestamping_->is_later(timestamp, start_time)) { - // The found item was timestamped after the start of the iteration, - // so it is save to remove it. - uint64_t expected = 0; - if (result->taken.load() == 0) { - if (result->taken.compare_exchange_weak( - expected, 1)) { - // Try to adjust the remove pointer. It does not matter if this - // CAS fails. - left_[buffer_index]->compare_exchange_weak( - old_left, (Item*)add_next_aba(result, old_left, 0)); - *element = result->data.load(); - return true; + if (result != NULL) + { + if (!timestamping_->is_later(timestamp, start_time)) + { + // The found item was timestamped after the start of the iteration, + // so it is save to remove it. + uint64_t expected = 0; + if (result->taken.load() == 0) + { + if (result->taken.compare_exchange_weak( + expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if this + // CAS fails. + left_[buffer_index]->compare_exchange_weak( + old_left, (Item *)add_next_aba(result, old_left, 0)); + *element = result->data.load(); + return true; + } + } } - } } - } - - *element = (T)NULL; - return !empty; - } - - bool try_remove_right(T *element, uint64_t *invocation_time) - { - // Initialize the data needed for the emptiness check. - uint64_t thread_id = scal::ThreadContext::get().thread_id(); - Item* *emptiness_check_left = - emptiness_check_left_[thread_id]; - Item* *emptiness_check_right = - emptiness_check_right_[thread_id]; - bool empty = true; - // Initialize the result pointer to NULL, which means that no - // element has been removed. - Item *result = NULL; - // Indicates the index which contains the youngest item. - uint64_t buffer_index = -1; - // Memory on the stack frame where timestamps of items can be stored - // temporarily. - uint64_t tmp_timestamp[2][2]; - // Index in the tmp_timestamp array whihc is not used at the moment. - uint64_t tmp_index = 1; - timestamping_->init_sentinel(tmp_timestamp[0]); - uint64_t *timestamp = tmp_timestamp[0]; - // Stores the value of the remove pointer of a thead-local buffer - // before the buffer is actually accessed. - Item* old_right = NULL; - - // Read the start time of the iteration. Items which were timestamped - // after the start time and inserted at the left are not removed. - uint64_t start_time[2]; - timestamping_->read_time(start_time); - // We start iterating over the thread-local lists at a random index. - uint64_t start = hwrand(); - // We iterate over all thead-local buffers - for (uint64_t i = 0; i < num_threads_; i++) { - - uint64_t tmp_buffer_index = (start + i) % num_threads_; - // We get the remove/insert pointer of the current thread-local buffer. - Item* tmp_right = right_[tmp_buffer_index]->load(); - // We get the youngest element from that thread-local buffer. - Item* item = get_right_item(tmp_buffer_index); - // If we found an element, we compare it to the youngest element - // we have found until now. - if (item != NULL) { - empty = false; - uint64_t *item_timestamp; - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; - - if (inserted_right(item) && !timestamping_->is_later(invocation_time, item_timestamp)) { - uint64_t expected = 0; - if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - right_[tmp_buffer_index]->compare_exchange_weak( - tmp_right, (Item*)add_next_aba(item, tmp_right, 0)); - *element = item->data.load(); - return true; - } else { - item = get_right_item(tmp_buffer_index); - if (item != NULL) { + + *element = (T)NULL; + return !empty; + } + + bool try_remove_right(T *element, uint64_t *invocation_time) + { + // Initialize the data needed for the emptiness check. + uint64_t thread_id = scal::ThreadContext::get().thread_id(); + Item **emptiness_check_left = + emptiness_check_left_[thread_id]; + Item **emptiness_check_right = + emptiness_check_right_[thread_id]; + bool empty = true; + // Initialize the result pointer to NULL, which means that no + // element has been removed. + Item *result = NULL; + // Indicates the index which contains the youngest item. + uint64_t buffer_index = -1; + // Memory on the stack frame where timestamps of items can be stored + // temporarily. + uint64_t tmp_timestamp[2][2]; + // Index in the tmp_timestamp array whihc is not used at the moment. + uint64_t tmp_index = 1; + timestamping_->init_sentinel(tmp_timestamp[0]); + uint64_t *timestamp = tmp_timestamp[0]; + // Stores the value of the remove pointer of a thead-local buffer + // before the buffer is actually accessed. + Item *old_right = NULL; + + // Read the start time of the iteration. Items which were timestamped + // after the start time and inserted at the left are not removed. + uint64_t start_time[2]; + timestamping_->read_time(start_time); + // We start iterating over the thread-local lists at a random index. + uint64_t start = hwrand(); + // We iterate over all thead-local buffers + for (uint64_t i = 0; i < num_threads_; i++) + { + + uint64_t tmp_buffer_index = (start + i) % num_threads_; + // We get the remove/insert pointer of the current thread-local buffer. + Item *tmp_right = right_[tmp_buffer_index]->load(); + // We get the youngest element from that thread-local buffer. + Item *item = get_right_item(tmp_buffer_index); + // If we found an element, we compare it to the youngest element + // we have found until now. + if (item != NULL) + { + empty = false; + uint64_t *item_timestamp; timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); item_timestamp = tmp_timestamp[tmp_index]; - } + + if (inserted_right(item) && !timestamping_->is_later(invocation_time, item_timestamp)) + { + uint64_t expected = 0; + if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + right_[tmp_buffer_index]->compare_exchange_weak( + tmp_right, (Item *)add_next_aba(item, tmp_right, 0)); + *element = item->data.load(); + return true; + } + else + { + item = get_right_item(tmp_buffer_index); + if (item != NULL) + { + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + } + } + } + + if (item != NULL && (result == NULL || is_more_right(item, item_timestamp, result, timestamp))) + { + // We found a new youngest element, so we remember it. + result = item; + buffer_index = tmp_buffer_index; + timestamp = item_timestamp; + tmp_index ^= 1; + old_right = tmp_right; + } + } + else + { + // No element was found, work on the emptiness check. + if (emptiness_check_right[tmp_buffer_index] != tmp_right) + { + empty = false; + emptiness_check_right[tmp_buffer_index] = + tmp_right; + } + Item *tmp_left = left_[tmp_buffer_index]->load(); + if (emptiness_check_left[tmp_buffer_index] != tmp_left) + { + empty = false; + emptiness_check_left[tmp_buffer_index] = + tmp_left; + } } - } - - if (item != NULL && (result == NULL || is_more_right(item, item_timestamp, result, timestamp))) { - // We found a new youngest element, so we remember it. - result = item; - buffer_index = tmp_buffer_index; - timestamp = item_timestamp; - tmp_index ^=1; - old_right = tmp_right; - } - } else { - // No element was found, work on the emptiness check. - if (emptiness_check_right[tmp_buffer_index] - != tmp_right) { - empty = false; - emptiness_check_right[tmp_buffer_index] = - tmp_right; - } - Item* tmp_left = left_[tmp_buffer_index]->load(); - if (emptiness_check_left[tmp_buffer_index] - != tmp_left) { - empty = false; - emptiness_check_left[tmp_buffer_index] = - tmp_left; - } } - } - if (result != NULL) { - if (!timestamping_->is_later(timestamp, start_time)) { - // The found item was timestamped after the start of the iteration, - // so it is save to remove it. - uint64_t expected = 0; - if (result->taken.load() == 0) { - if (result->taken.compare_exchange_weak( - expected, 1)) { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - right_[buffer_index]->compare_exchange_weak( - old_right, (Item*)add_next_aba(result, old_right, 0)); - *element = result->data.load(); - return true; + if (result != NULL) + { + if (!timestamping_->is_later(timestamp, start_time)) + { + // The found item was timestamped after the start of the iteration, + // so it is save to remove it. + uint64_t expected = 0; + if (result->taken.load() == 0) + { + if (result->taken.compare_exchange_weak( + expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + right_[buffer_index]->compare_exchange_weak( + old_right, (Item *)add_next_aba(result, old_right, 0)); + *element = result->data.load(); + return true; + } + } } - } } - } - *element = (T)NULL; - return !empty; - } + *element = (T)NULL; + return !empty; + } }; diff --git a/cds/container/ts_timestamp.h b/cds/container/ts_timestamp.h index 2d0291ded..4bea9fe3d 100644 --- a/cds/container/ts_timestamp.h +++ b/cds/container/ts_timestamp.h @@ -3,149 +3,183 @@ inline uint64_t get_hwptime(void) { uint64_t aux; - uint64_t rax,rdx; - asm volatile ( "rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); + uint64_t rax, rdx; + asm volatile ("rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); return (rdx << 32) + rax; } -inline uint64_t get_hwtime(void) { - unsigned int hi, lo; - __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); - return ((uint64_t) lo) | (((uint64_t) hi) << 32); +inline uint64_t get_hwtime(void) +{ + unsigned int hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t)lo) | (((uint64_t)hi) << 32); } -class HardwareTimestamp { - public: - inline void initialize(uint64_t delay, uint64_t num_threads) { +class HardwareTimestamp +{ +public: + inline void initialize(uint64_t delay, uint64_t num_threads) + { } - inline void init_sentinel(uint64_t *result) { - result[0] = 0; + inline void init_sentinel(uint64_t *result) + { + result[0] = 0; } - inline void init_sentinel_atomic(std::atomic *result) { - result[0].store(0); + inline void init_sentinel_atomic(std::atomic *result) + { + result[0].store(0); } - inline void init_top_atomic(std::atomic *result) { - result[0].store(UINT64_MAX); + inline void init_top_atomic(std::atomic *result) + { + result[0].store(UINT64_MAX); } - inline void init_top(uint64_t *result) { - result[0] = UINT64_MAX; + inline void init_top(uint64_t *result) + { + result[0] = UINT64_MAX; } - inline void load_timestamp(uint64_t *result, std::atomic *source) { - result[0] = source[0].load(); + inline void load_timestamp(uint64_t *result, std::atomic *source) + { + result[0] = source[0].load(); } - inline void set_timestamp(std::atomic *result) { - result[0].store(get_hwptime()); + inline void set_timestamp(std::atomic *result) + { + result[0].store(get_hwptime()); } - inline void read_time(uint64_t *result) { - result[0] = get_hwptime(); + inline void read_time(uint64_t *result) + { + result[0] = get_hwptime(); } - inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) { - return timestamp2[0] < timestamp1[0]; + inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) + { + return timestamp2[0] < timestamp1[0]; } }; -class HardwareIntervalTimestamp { - private: +class HardwareIntervalTimestamp +{ +private: uint64_t delay_; - public: - inline void initialize(uint64_t delay, uint64_t num_threads) { - delay_ = delay; +public: + inline void initialize(uint64_t delay, uint64_t num_threads) + { + delay_ = delay; } - inline void init_sentinel(uint64_t *result) { - result[0] = 0; - result[1] = 0; + inline void init_sentinel(uint64_t *result) + { + result[0] = 0; + result[1] = 0; } - inline void init_sentinel_atomic(std::atomic *result) { - result[0].store(0); - result[1].store(0); + inline void init_sentinel_atomic(std::atomic *result) + { + result[0].store(0); + result[1].store(0); } - inline void init_top_atomic(std::atomic *result) { - result[0].store(UINT64_MAX); - result[1].store(UINT64_MAX); + inline void init_top_atomic(std::atomic *result) + { + result[0].store(UINT64_MAX); + result[1].store(UINT64_MAX); } - inline void init_top(uint64_t *result) { - result[0] = UINT64_MAX; - result[1] = UINT64_MAX; + inline void init_top(uint64_t *result) + { + result[0] = UINT64_MAX; + result[1] = UINT64_MAX; } - inline void load_timestamp(uint64_t *result, std::atomic *source) { - result[0] = source[0].load(); - result[1] = source[1].load(); + inline void load_timestamp(uint64_t *result, std::atomic *source) + { + result[0] = source[0].load(); + result[1] = source[1].load(); } - inline void set_timestamp(std::atomic *result) { - result[0].store(get_hwptime()); - uint64_t wait = get_hwtime() + delay_; - while (get_hwtime() < wait) {} - result[1].store(get_hwptime()); + inline void set_timestamp(std::atomic *result) + { + result[0].store(get_hwptime()); + uint64_t wait = get_hwtime() + delay_; + while (get_hwtime() < wait) + { + } + result[1].store(get_hwptime()); } - inline void read_time(uint64_t *result) { - result[0] = get_hwptime(); - result[1] = result[0]; + inline void read_time(uint64_t *result) + { + result[0] = get_hwptime(); + result[1] = result[0]; } - inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) { - return timestamp2[1] < timestamp1[0]; + inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) + { + return timestamp2[1] < timestamp1[0]; } }; -class AtomicCounterTimestamp { - private: +class AtomicCounterTimestamp +{ +private: std::atomic *clock_; - public: - inline void initialize(uint64_t delay, uint64_t num_threads) { - clock_ = new std::atomic(); - clock_->store(1); +public: + inline void initialize(uint64_t delay, uint64_t num_threads) + { + clock_ = new std::atomic(); + clock_->store(1); } - inline void init_sentinel(uint64_t *result) { - result[0] = 0; + inline void init_sentinel(uint64_t *result) + { + result[0] = 0; } - inline void init_sentinel_atomic(std::atomic *result) { - result[0].store(0); + inline void init_sentinel_atomic(std::atomic *result) + { + result[0].store(0); } - inline void init_top_atomic(std::atomic *result) { - result[0].store(UINT64_MAX); + inline void init_top_atomic(std::atomic *result) + { + result[0].store(UINT64_MAX); } - inline void init_top(uint64_t *result) { - result[0] = UINT64_MAX; + inline void init_top(uint64_t *result) + { + result[0] = UINT64_MAX; } - inline void load_timestamp(uint64_t *result, std::atomic *source) { - result[0] = source[0].load(); + inline void load_timestamp(uint64_t *result, std::atomic *source) + { + result[0] = source[0].load(); } - inline void set_timestamp(std::atomic *result) { - result[0].store(clock_->fetch_add(1)); + inline void set_timestamp(std::atomic *result) + { + result[0].store(clock_->fetch_add(1)); } - inline void set_timestamp_local(uint64_t *result) { - result[0] = clock_->fetch_add(1); + inline void set_timestamp_local(uint64_t *result) + { + result[0] = clock_->fetch_add(1); } - inline void read_time(uint64_t *result) { - result[0] = clock_->load(); + inline void read_time(uint64_t *result) + { + result[0] = clock_->load(); } - inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) { - return timestamp2[0] < timestamp1[0]; + inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) + { + return timestamp2[0] < timestamp1[0]; } }; \ No newline at end of file From 01819342cbfa8beb834838819c992d20c9667b02 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Sun, 23 Dec 2018 23:52:59 +0300 Subject: [PATCH 19/29] add clear, size and empty methods --- cds/container/threadcontext.h | 2 +- cds/container/ts_deque.h | 29 +++++++++++++++++++++++++++-- cds/container/ts_deque_buffer.h | 8 ++++---- cds/container/ts_timestamp.h | 2 +- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cds/container/threadcontext.h b/cds/container/threadcontext.h index 046c4b3f4..191abfaf6 100644 --- a/cds/container/threadcontext.h +++ b/cds/container/threadcontext.h @@ -58,4 +58,4 @@ void ThreadContext::prepare(uint64_t num_threads) context->thread_id_ = i; contexts[i] = context; } -} \ No newline at end of file +} diff --git a/cds/container/ts_deque.h b/cds/container/ts_deque.h index fa027f5c2..e74a0d2ff 100644 --- a/cds/container/ts_deque.h +++ b/cds/container/ts_deque.h @@ -8,6 +8,7 @@ class TSDeque private: TSDequeBuffer *buffer_; Timestamp *timestamping_; + item_counter item_counter_; public: TSDeque(uint64_t num_threads, uint64_t delay) @@ -22,6 +23,12 @@ class TSDeque buffer_->initialize(num_threads, timestamping_); } + ~TSDeque() + { + delete timestamping_; + delete buffer_; + } + bool insert_left(T element) { std::atomic *item = buffer_->insert_left(element); @@ -29,6 +36,7 @@ class TSDeque // and then assigned to the item. The operation may not be executed // atomically. timestamping_->set_timestamp(item); + item_counter_++; return true; } @@ -39,6 +47,7 @@ class TSDeque // and then assigned to the item. The operation may not be executed // atomically. timestamping_->set_timestamp(item); + item_counter_++; return true; } @@ -50,9 +59,9 @@ class TSDeque timestamping_->read_time(invocation_time); while (buffer_->try_remove_left(element, invocation_time)) { - if (*element != (T)NULL) { + item_counter_--; return true; } } @@ -68,13 +77,29 @@ class TSDeque timestamping_->read_time(invocation_time); while (buffer_->try_remove_right(element, invocation_time)) { - if (*element != (T)NULL) { + item_counter_--; return true; } } // The deque was empty, return false. return false; } + + void clear() + { + T item; + while (remove_right(&item)) {} + } + + bool empty() const + { + return item_counter_ == 0; + } + + size_t size() const + { + return item_counter_; + } }; diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index bee83fc6d..cb7555145 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -187,7 +187,7 @@ class TSDequeBuffer inline std::atomic *insert_left(T element) { - uint64_t thread_id = scal::ThreadContext::get().thread_id(); + uint64_t thread_id = ThreadContext::get().thread_id(); // Create a new item. Item *new_item = new Item(); @@ -237,7 +237,7 @@ class TSDequeBuffer inline std::atomic *insert_right(T element) { - uint64_t thread_id = scal::ThreadContext::get().thread_id(); + uint64_t thread_id = ThreadContext::get().thread_id(); // Create a new item. Item *new_item = new Item(); @@ -348,7 +348,7 @@ class TSDequeBuffer bool try_remove_left(T *element, uint64_t *invocation_time) { // Initialize the data needed for the emptiness check. - uint64_t thread_id = scal::ThreadContext::get().thread_id(); + uint64_t thread_id = ThreadContext::get().thread_id(); Item **emptiness_check_left = emptiness_check_left_[thread_id]; Item **emptiness_check_right = @@ -495,7 +495,7 @@ class TSDequeBuffer bool try_remove_right(T *element, uint64_t *invocation_time) { // Initialize the data needed for the emptiness check. - uint64_t thread_id = scal::ThreadContext::get().thread_id(); + uint64_t thread_id = ThreadContext::get().thread_id(); Item **emptiness_check_left = emptiness_check_left_[thread_id]; Item **emptiness_check_right = diff --git a/cds/container/ts_timestamp.h b/cds/container/ts_timestamp.h index 4bea9fe3d..3c7665e22 100644 --- a/cds/container/ts_timestamp.h +++ b/cds/container/ts_timestamp.h @@ -182,4 +182,4 @@ class AtomicCounterTimestamp { return timestamp2[0] < timestamp1[0]; } -}; \ No newline at end of file +}; From c4ecac4423e60c5262b427c4a47430b2a8cea90d Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Mon, 24 Dec 2018 23:45:25 +0300 Subject: [PATCH 20/29] prepare to pull request --- cds/container/threadcontext.h | 61 -- cds/container/ts_deque.h | 297 ++++++--- cds/container/ts_deque_buffer.h | 1040 ++++++++++++++++--------------- cds/container/ts_timestamp.h | 299 ++++----- 4 files changed, 892 insertions(+), 805 deletions(-) delete mode 100644 cds/container/threadcontext.h diff --git a/cds/container/threadcontext.h b/cds/container/threadcontext.h deleted file mode 100644 index 191abfaf6..000000000 --- a/cds/container/threadcontext.h +++ /dev/null @@ -1,61 +0,0 @@ -#include -#include - -class ThreadContext -{ -public: - static ThreadContext &get(); - static void prepare(uint64_t num_threads); - static void assign_context(); - - inline uint64_t thread_id() - { - return thread_id_; - } - -private: - static constexpr uint64_t kMaxThreads = 1024; - static uint64_t global_thread_id_cnt; - static ThreadContext *contexts[kMaxThreads]; - static pthread_key_t threadcontext_key; - - ThreadContext() {} - - uint64_t thread_id_; -}; - -uint64_t ThreadContext::global_thread_id_cnt = 0; -pthread_key_t ThreadContext::threadcontext_key; -ThreadContext *ThreadContext::contexts[kMaxThreads]; - -ThreadContext &ThreadContext::get() -{ - if (pthread_getspecific(threadcontext_key) == NULL) - { - assign_context(); - } - ThreadContext *context = static_cast( - pthread_getspecific(threadcontext_key)); - return *context; -} - -void ThreadContext::assign_context() -{ - uint64_t thread_id = __sync_fetch_and_add(&global_thread_id_cnt, 1); - if (pthread_setspecific(threadcontext_key, contexts[thread_id])) - { - fprintf(stderr, "%s: pthread_setspecific failed\n", __func__); - exit(EXIT_FAILURE); - } -} - -void ThreadContext::prepare(uint64_t num_threads) -{ - pthread_key_create(&threadcontext_key, NULL); - for (uint64_t i = 0; i < num_threads; i++) - { - ThreadContext *context = new ThreadContext(); - context->thread_id_ = i; - contexts[i] = context; - } -} diff --git a/cds/container/ts_deque.h b/cds/container/ts_deque.h index e74a0d2ff..5ac7722a1 100644 --- a/cds/container/ts_deque.h +++ b/cds/container/ts_deque.h @@ -1,105 +1,228 @@ +#ifndef CDSLIB_CONTAINER_TS_DEQUE_H +#define CDSLIB_CONTAINER_TS_DEQUE_H + #include -#include "ts_deque_buffer.h" -#include "threadcontext.h" - -template -class TSDeque -{ -private: - TSDequeBuffer *buffer_; - Timestamp *timestamping_; - item_counter item_counter_; - -public: - TSDeque(uint64_t num_threads, uint64_t delay) - { - ThreadContext::prepare(num_threads); - ThreadContext::assign_context(); +#include +#include - timestamping_ = new Timestamp(); - timestamping_->initialize(delay, num_threads); +namespace cds { namespace container { - buffer_ = new TSDequeBuffer(); - buffer_->initialize(num_threads, timestamping_); - } + /// TSDeque related definitions + /** @ingroup cds_nonintrusive_helper + */ + namespace tsdeque { + /// TSDeque default type traits + struct traits + { + /// Item counting feature; by default, disabled. Use \p cds::atomicity::item_counter to enable item counting + typedef cds::atomicity::empty_item_counter item_counter; - ~TSDeque() - { - delete timestamping_; - delete buffer_; - } + /// Random engine to generate a random position in array of thread-local buffers + typedef opt::v::c_rand random_engine; + }; - bool insert_left(T element) - { - std::atomic *item = buffer_->insert_left(element); - // In the set_timestamp operation first a new timestamp is acquired - // and then assigned to the item. The operation may not be executed - // atomically. - timestamping_->set_timestamp(item); - item_counter_++; - return true; - } - - bool insert_right(T element) - { - std::atomic *item = buffer_->insert_right(element); - // In the set_timestamp operation first a new timestamp is acquired - // and then assigned to the item. The operation may not be executed - // atomically. - timestamping_->set_timestamp(item); - item_counter_++; - return true; - } - - bool remove_left(T *element) + /// Metafunction converting option list to \p tsdeque::traits + /** + Supported \p Options are: + - opt::item_counter - the type of item counting feature. + Default is \p cds::atomicity::empty_item_counter (item counting disabled). + - opt::random_engine - a random engine to generate a random position in array of thread-local buffers. + Default is \p opt::v::c_rand. + + Example: declare \p %TSDeque with item counting + \code + typedef cds::container::TSDeque< Foo, + typename cds::container::tsdeque::make_traits< + cds::opt::item_counter< cds::atomicity::item_counter > + >::type + > myDeque; + \endcode + */ + template + struct make_traits { +# ifdef CDS_DOXYGEN_INVOKED + typedef implementation_defined type; ///< Metafunction result +# else + typedef typename cds::opt::make_options< + typename cds::opt::find_type_traits< traits, Options... >::type + , Options... + >::type type; +# endif + }; + + } // namespace tsdeque + + /// Fast Concurrent Deque Through Explicit Timestamping + /** @ingroup cds_nonintrusive_deque + + Source + - [2014] Mike Dodds, Andreas Haas, Christoph M. Kirsch + "Fast Concurrent Data-Structures Through Explicit Timestamping" + + Template arguments + - \p T - value type to be stored in the deque + - \p Timestamp - the way to acquire timestamps + - \p Traits - deque traits, default is \p tsdeque::traits. You can use \p tsdeque::make_traits + metafunction to make your traits or just derive your traits from \p %tsdeque::traits: + \code + struct myTraits: public cds::container::tsdeque::traits { + typedef cds::atomicity::item_counter item_counter; + }; + typedef cds::container::TSDeque< Foo, Timestamp, myTraits > myDeque; + + // Equivalent make_traits example: + typedef cds::container::TSDeque< Foo, Timestamp + typename cds::container::tsdeque::make_traits< + cds::opt::item_counter< cds::atomicity::item_counter > + >::type + > myDeque; + \endcode + */ + template + class TSDeque { - // Read the invocation time of this operation, needed for the - // elimination optimization. - uint64_t invocation_time[2]; - timestamping_->read_time(invocation_time); - while (buffer_->try_remove_left(element, invocation_time)) + public: + typedef T value_type; ///< Type of value to be stored in the deque + typedef Timestamp timestamp; ///< Algorithm of acquiring timestamps + typedef Traits traits; ///< Deque traits + + typedef typename traits::item_counter item_counter; ///< Item counting policy used + typedef typename traits::random_engine random_engine; ///< Random engine used + + private: + TSDequeBuffer *buffer_; + timestamp *timestamping_; + item_counter item_counter_; + + public: + TSDeque(uint64_t num_threads, uint64_t delay) + { + timestamping_ = new timestamp(); + timestamping_->initialize(delay, num_threads); + + buffer_ = new TSDequeBuffer(); + buffer_->initialize(num_threads, timestamping_); + } + + ~TSDeque() + { + clear(); + + delete timestamping_; + delete buffer_; + } + + /// Inserts a new element at the left end of the deque container + /** + The function always returns \p true + */ + bool insert_left(value_type element) + { + std::atomic *item = buffer_->insert_left(element); + // In the set_timestamp operation first a new timestamp is acquired + // and then assigned to the item. The operation may not be executed + // atomically. + timestamping_->set_timestamp(item); + ++item_counter_; + return true; + } + + /// Inserts a new element at the right end of the deque container + /** + The function always returns \p true + */ + bool insert_right(value_type element) { - if (*element != (T)NULL) + std::atomic *item = buffer_->insert_right(element); + // In the set_timestamp operation first a new timestamp is acquired + // and then assigned to the item. The operation may not be executed + // atomically. + timestamping_->set_timestamp(item); + ++item_counter_; + return true; + } + + /// Removes the element from the left end of the deque container + /** + The function returns \p false if the deque is empty, \p true otherwise. + */ + bool remove_left(value_type *element) + { + // Read the invocation time of this operation, needed for the + // elimination optimization. + uint64_t invocation_time[2]; + timestamping_->read_time(invocation_time); + while (buffer_->try_remove_left(element, invocation_time)) { - item_counter_--; - return true; + if (element != NULL) + { + --item_counter_; + return true; + } } + // The deque was empty, return false. + return false; } - // The deque was empty, return false. - return false; - } - bool remove_right(T *element) - { - // Read the invocation time of this operation, needed for the - // elimination optimization. - uint64_t invocation_time[2]; - timestamping_->read_time(invocation_time); - while (buffer_->try_remove_right(element, invocation_time)) + /// Removes the element from the right end of the deque container + /** + The function returns \p false if the deque is empty, \p true otherwise. + */ + bool remove_right(value_type *element) { - if (*element != (T)NULL) + // Read the invocation time of this operation, needed for the + // elimination optimization. + uint64_t invocation_time[2]; + timestamping_->read_time(invocation_time); + while (buffer_->try_remove_right(element, invocation_time)) { - item_counter_--; - return true; + if (element != NULL) + { + --item_counter_; + return true; + } } + // The deque was empty, return false. + return false; } - // The deque was empty, return false. - return false; - } - void clear() - { - T item; - while (remove_right(&item)) {} - } + /// Clears the deque (non-atomic) + /** + The function erases all items from the deque. - bool empty() const - { - return item_counter_ == 0; - } + The function is not atomic. It cleans up the deque and then resets the item counter to zero. + If there are a thread that performs insertion while \p clear is working the result is undefined in general case: + empty() may return \p true but the deque may contain item(s). + Therefore, \p clear may be used only for debugging purposes. + */ + void clear() + { + value_type item; + while (remove_right(&item)) {} + item_counter_.reset(); + } - size_t size() const - { - return item_counter_; - } -}; + /// Checks if the deque is empty + /** + @warning If you use \p atomicity::empty_item_counter in \p traits::item_counter, + the function always returns \p true. + */ + bool empty() const + { + return size() == 0; + } + + /// Returns item count in the deque + /** + @warning If you use \p atomicity::empty_item_counter in \p traits::item_counter, + the function always returns 0. + */ + size_t size() const + { + return item_counter_; + } + }; + +}} // namespace cds::container + +#endif // #ifndef CDSLIB_CONTAINER_TS_DEQUE_H diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index cb7555145..b1d74e6ab 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -1,621 +1,637 @@ +#ifndef CDSLIB_CONTAINER_TS_DEQUE_BUFFER_H +#define CDSLIB_CONTAINER_TS_DEQUE_BUFFER_H + #include #include #include -#include "threadcontext.h" - -inline uint64_t rdtsc() -{ - unsigned int hi, lo; - __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); - return ((uint64_t)lo) | (((uint64_t)hi) << 32); -} - -inline uint64_t hwrand() -{ - return (rdtsc() >> 6); -} - -template -class TSDequeBuffer -{ -private: - typedef struct Item - { - std::atomic left; - std::atomic right; - std::atomic taken; - std::atomic data; - std::atomic timestamp[2]; - // Insertion index, needed for the termination condition in - // get_left_item. Items inserted at the left get negative - // indices, items inserted at the right get positive indices. - std::atomic index; - } Item; - - // The number of threads. - uint64_t num_threads_; - std::atomic **left_; - std::atomic **right_; - int64_t **next_index_; - // The pointers for the emptiness check. - Item ***emptiness_check_left_; - Item ***emptiness_check_right_; - TimeStamp *timestamping_; - - // Helper function to remove the ABA counter from a pointer. - void *get_aba_free_pointer(void *pointer) - { - uint64_t result = (uint64_t)pointer; - result &= 0xfffffffffffffff8; - return (void *)result; - } - - // Helper function which retrieves the ABA counter of a pointer old - // and sets this ABA counter + increment to the pointer pointer. - void *add_next_aba(void *pointer, void *old, uint64_t increment) - { - uint64_t aba = (uint64_t)old; - aba += increment; - aba &= 0x7; - uint64_t result = (uint64_t)pointer; - result = (result & 0xfffffffffffffff8) | aba; - return (void *)((result & 0xffffffffffffff8) | aba); - } - - // Returns the leftmost not-taken item from the thread-local list - // indicated by thread_id. - Item *get_left_item(uint64_t thread_id) - { - - // Read the item pointed to by the right pointer. The iteration through - // the linked list can stop at that item. - Item *old_right = right_[thread_id]->load(); - Item *right = (Item *)get_aba_free_pointer(old_right); - int64_t threshold = right->index.load(); +#include - // Read the leftmost item. - Item *result = (Item *)get_aba_free_pointer(left_[thread_id]->load()); +namespace cds { namespace container { - // We start at the left pointer and iterate to the right until we - // find the first item which has not been taken yet. - while (true) + template + class TSDequeBuffer + { + private: + typedef struct Item { - // We reached a node further right than the original right-most - // node. We do not have to search any further to the right, we - // will not take the element anyways. - if (result->index.load() > threshold) - { - return NULL; - } - // We found a good node, return it. - if (result->taken.load() == 0) - { - return result; - } - // We have reached the end of the list and found nothing, so we - // return NULL. - if (result->right.load() == result) + std::atomic left; + std::atomic right; + std::atomic taken; + std::atomic data; + std::atomic timestamp[2]; + // Insertion index, needed for the termination condition in + // get_left_item. Items inserted at the left get negative + // indices, items inserted at the right get positive indices. + std::atomic index; + } Item; + + uint64_t num_threads_; + std::atomic **left_; + std::atomic **right_; + int64_t **next_index_; + // The pointers for the emptiness check. + Item ***emptiness_check_left_; + Item ***emptiness_check_right_; + + TimeStamp *timestamping_; + + RandomEngine random_engine_; + + std::atomic thread_id_counter_; + boost::thread_specific_ptr thread_id_; + + int get_thread_id() + { + int* temp = thread_id_.get(); + if (temp == NULL) { - return NULL; - } - result = result->right.load(); - } - } - - // Returns the rightmost not-taken item from the thread-local list - // indicated by thread_id. - Item *get_right_item(uint64_t thread_id) - { + int index = thread_id_counter_.load(); + if(index >= num_threads_) + { + return -1; + } - // Read the item pointed to by the left pointer. The iteration through - // the linked list can stop at that item. - Item *old_left = left_[thread_id]->load(); - Item *left = (Item *)get_aba_free_pointer(old_left); - int64_t threshold = left->index.load(); + while(!thread_id_counter_.compare_exchange_strong(index, index + 1)) + { + index = thread_id_counter_.load(); + } - Item *result = (Item *)get_aba_free_pointer(right_[thread_id]->load()); + thread_id_.reset(new int (index)); + return index; + } + return *temp; + } - // We start at the right pointer and iterate to the left until we - // find the first item which has not been taken yet. - while (true) + // Helper function to remove the ABA counter from a pointer. + void *get_aba_free_pointer(void *pointer) { - // We reached a node further left than the original left-most - // node. We do not have to search any further to the left, we - // will not take the element anyways. - if (result->index.load() < threshold) - { - return NULL; - } - // We found a good node, return it. - if (result->taken.load() == 0) - { - return result; - } - // We have reached the end of the list and found nothing, so we - // return NULL. - if (result->left.load() == result) - { - return NULL; - } - result = result->left.load(); + uint64_t result = (uint64_t)pointer; + result &= 0xfffffffffffffff8; + return (void *)result; } - } -public: - void initialize(uint64_t num_threads, TimeStamp *timestamping) - { + // Helper function which retrieves the ABA counter of a pointer old + // and sets this ABA counter + increment to the pointer pointer. + void *add_next_aba(void *pointer, void *old, uint64_t increment) + { + uint64_t aba = (uint64_t)old; + aba += increment; + aba &= 0x7; + uint64_t result = (uint64_t)pointer; + result = (result & 0xfffffffffffffff8) | aba; + return (void *)((result & 0xffffffffffffff8) | aba); + } - num_threads_ = num_threads; - timestamping_ = timestamping; + // Returns the leftmost not-taken item from the thread-local list + // indicated by thread_id. + Item *get_left_item(uint64_t thread_id) + { + // Read the item pointed to by the right pointer. The iteration through + // the linked list can stop at that item. + Item *old_right = right_[thread_id]->load(); + Item *right = (Item *)get_aba_free_pointer(old_right); + int64_t threshold = right->index.load(); - left_ = new std::atomic *[num_threads_]; + // Read the leftmost item. + Item *result = (Item *)get_aba_free_pointer(left_[thread_id]->load()); - right_ = new std::atomic *[num_threads_]; + // We start at the left pointer and iterate to the right until we + // find the first item which has not been taken yet. + while (true) + { + // We reached a node further right than the original right-most + // node. We do not have to search any further to the right, we + // will not take the element anyways. + if (result->index.load() > threshold) + { + return NULL; + } + // We found a good node, return it. + if (result->taken.load() == 0) + { + return result; + } + // We have reached the end of the list and found nothing, so we + // return NULL. + if (result->right.load() == result) + { + return NULL; + } + result = result->right.load(); + } + } - next_index_ = new int64_t *[num_threads_]; + // Returns the rightmost not-taken item from the thread-local list + // indicated by thread_id. + Item *get_right_item(uint64_t thread_id) + { + // Read the item pointed to by the left pointer. The iteration through + // the linked list can stop at that item. + Item *old_left = left_[thread_id]->load(); + Item *left = (Item *)get_aba_free_pointer(old_left); + int64_t threshold = left->index.load(); - emptiness_check_left_ = new Item **[num_threads_]; + Item *result = (Item *)get_aba_free_pointer(right_[thread_id]->load()); - emptiness_check_right_ = new Item **[num_threads_]; + // We start at the right pointer and iterate to the left until we + // find the first item which has not been taken yet. + while (true) + { + // We reached a node further left than the original left-most + // node. We do not have to search any further to the left, we + // will not take the element anyways. + if (result->index.load() < threshold) + { + return NULL; + } + // We found a good node, return it. + if (result->taken.load() == 0) + { + return result; + } + // We have reached the end of the list and found nothing, so we + // return NULL. + if (result->left.load() == result) + { + return NULL; + } + result = result->left.load(); + } + } - for (uint64_t i = 0; i < num_threads_; i++) + public: + void initialize(uint64_t num_threads, TimeStamp *timestamping) { + thread_id_counter_.store(0); - left_[i] = new std::atomic(); + num_threads_ = num_threads; + timestamping_ = timestamping; - right_[i] = new std::atomic(); + left_ = new std::atomic *[num_threads_]; + right_ = new std::atomic *[num_threads_]; - next_index_[i] = new int64_t(); + next_index_ = new int64_t *[num_threads_]; - // Add a sentinal node. - Item *new_item = new Item(); - timestamping_->init_sentinel_atomic(new_item->timestamp); - new_item->data.store(0); - new_item->taken.store(1); - new_item->left.store(new_item); - new_item->right.store(new_item); - new_item->index.store(0); - left_[i]->store(new_item); - right_[i]->store(new_item); - *next_index_[i] = 1; + emptiness_check_left_ = new Item **[num_threads_]; + emptiness_check_right_ = new Item **[num_threads_]; - emptiness_check_left_[i] = new Item *[num_threads_]; - - emptiness_check_right_[i] = new Item *[num_threads_]; + for (uint64_t i = 0; i < num_threads_; i++) + { + left_[i] = new std::atomic(); + right_[i] = new std::atomic(); + + next_index_[i] = new int64_t(); + + // Add a sentinal node. + Item *new_item = new Item(); + timestamping_->init_sentinel_atomic(new_item->timestamp); + new_item->data.store(0); + new_item->taken.store(1); + new_item->left.store(new_item); + new_item->right.store(new_item); + new_item->index.store(0); + left_[i]->store(new_item); + right_[i]->store(new_item); + *next_index_[i] = 1; + + emptiness_check_left_[i] = new Item *[num_threads_]; + emptiness_check_right_[i] = new Item *[num_threads_]; + } } - } - inline std::atomic *insert_left(T element) - { - uint64_t thread_id = ThreadContext::get().thread_id(); - - // Create a new item. - Item *new_item = new Item(); - timestamping_->init_top_atomic(new_item->timestamp); - new_item->data.store(element); - new_item->taken.store(0); - new_item->left.store(new_item); - // Items inserted at the left get negative indices. Thereby the - // order of items in the thread-local lists correspond with the - // order of indices, and we can use the sign of the index to - // determine on which side an item has been inserted. - new_item->index = -((*next_index_[thread_id])++); - - // Determine leftmost not-taken item in the list. The new item is - // inserted to the left of that item. - Item *old_left = left_[thread_id]->load(); - - Item *left = (Item *)get_aba_free_pointer(old_left); - while (left->right.load() != left && left->taken.load()) + inline std::atomic *insert_left(T element) { - left = left->right.load(); - } + uint64_t thread_id = get_thread_id(); - if (left->taken.load() && left->right.load() == left) - { - // The buffer is empty. We have to increase the aba counter of the - // right pointer too to guarantee that a pending right-pointer - // update of a remove operation does not make the left and the - // right pointer point to different lists. + // Create a new item. + Item *new_item = new Item(); + timestamping_->init_top_atomic(new_item->timestamp); + new_item->data.store(element); + new_item->taken.store(0); + new_item->left.store(new_item); + // Items inserted at the left get negative indices. Thereby the + // order of items in the thread-local lists correspond with the + // order of indices, and we can use the sign of the index to + // determine on which side an item has been inserted. + new_item->index = -((*next_index_[thread_id])++); + + // Determine leftmost not-taken item in the list. The new item is + // inserted to the left of that item. + Item *old_left = left_[thread_id]->load(); - left = (Item *)get_aba_free_pointer(old_left); - left->right.store(left); - Item *old_right = right_[thread_id]->load(); - right_[thread_id]->store((Item *)add_next_aba(left, old_right, 1)); - } + Item *left = (Item *)get_aba_free_pointer(old_left); + while (left->right.load() != left && left->taken.load()) + { + left = left->right.load(); + } - // Add the new item to the list. - new_item->right.store(left); - left->left.store(new_item); - left_[thread_id]->store( - (Item *)add_next_aba(new_item, old_left, 1)); + if (left->taken.load() && left->right.load() == left) + { + // The buffer is empty. We have to increase the aba counter of the + // right pointer too to guarantee that a pending right-pointer + // update of a remove operation does not make the left and the + // right pointer point to different lists. + + left = (Item *)get_aba_free_pointer(old_left); + left->right.store(left); + Item *old_right = right_[thread_id]->load(); + right_[thread_id]->store((Item *)add_next_aba(left, old_right, 1)); + } - // Return a pointer to the timestamp location of the item so that a - // timestamp can be added. - return new_item->timestamp; - } + // Add the new item to the list. + new_item->right.store(left); + left->left.store(new_item); + left_[thread_id]->store( + (Item *)add_next_aba(new_item, old_left, 1)); - inline std::atomic *insert_right(T element) - { - uint64_t thread_id = ThreadContext::get().thread_id(); - - // Create a new item. - Item *new_item = new Item(); - timestamping_->init_top_atomic(new_item->timestamp); - new_item->data.store(element); - new_item->taken.store(0); - new_item->right.store(new_item); - new_item->index = (*next_index_[thread_id])++; - - // Determine the rightmost not-taken item in the list. The new item is - // inserted to the right of that item. - Item *old_right = right_[thread_id]->load(); - - Item *right = (Item *)get_aba_free_pointer(old_right); - while (right->left.load() != right && right->taken.load()) - { - right = right->left.load(); + // Return a pointer to the timestamp location of the item so that a + // timestamp can be added. + return new_item->timestamp; } - if (right->taken.load() && right->left.load() == right) + inline std::atomic *insert_right(T element) { - // The buffer is empty. We have to increase the aba counter of the - // left pointer too to guarantee that a pending left-pointer - // update of a remove operation does not make the left and the - // right pointer point to different lists. - right = (Item *)get_aba_free_pointer(old_right); - right->left.store(right); - Item *old_left = left_[thread_id]->load(); - left_[thread_id]->store((Item *)add_next_aba(right, old_left, 1)); - } - - // Add the new item to the list. - new_item->left.store(right); - right->right.store(new_item); - right_[thread_id]->store((Item *)add_next_aba(new_item, old_right, 1)); - - // Return a pointer to the timestamp location of the item so that a - // timestamp can be added. - return new_item->timestamp; - } + uint64_t thread_id = get_thread_id(); - // Helper function which returns true if the item was inserted at the left. - inline bool inserted_left(Item *item) - { - return item->index.load() < 0; - } + // Create a new item. + Item *new_item = new Item(); + timestamping_->init_top_atomic(new_item->timestamp); + new_item->data.store(element); + new_item->taken.store(0); + new_item->right.store(new_item); + new_item->index = (*next_index_[thread_id])++; - // Helper function which returns true if the item was inserted at the right. - inline bool inserted_right(Item *item) - { - return item->index.load() > 0; - } + // Determine the rightmost not-taken item in the list. The new item is + // inserted to the right of that item. + Item *old_right = right_[thread_id]->load(); - // Helper function which returns true if item1 is more left than item2. - inline bool is_more_left(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) - { - if (inserted_left(item2)) - { - if (inserted_left(item1)) + Item *right = (Item *)get_aba_free_pointer(old_right); + while (right->left.load() != right && right->taken.load()) { - return timestamping_->is_later(timestamp1, timestamp2); + right = right->left.load(); } - else + + if (right->taken.load() && right->left.load() == right) { - return false; + // The buffer is empty. We have to increase the aba counter of the + // left pointer too to guarantee that a pending left-pointer + // update of a remove operation does not make the left and the + // right pointer point to different lists. + right = (Item *)get_aba_free_pointer(old_right); + right->left.store(right); + Item *old_left = left_[thread_id]->load(); + left_[thread_id]->store((Item *)add_next_aba(right, old_left, 1)); } + + // Add the new item to the list. + new_item->left.store(right); + right->right.store(new_item); + right_[thread_id]->store((Item *)add_next_aba(new_item, old_right, 1)); + + // Return a pointer to the timestamp location of the item so that a + // timestamp can be added. + return new_item->timestamp; } - else + + // Helper function which returns true if the item was inserted at the left. + inline bool inserted_left(Item *item) { - if (inserted_left(item1)) - { - return true; - } - else - { - return timestamping_->is_later(timestamp2, timestamp1); - } + return item->index.load() < 0; } - } - // Helper function which returns true if item1 is more right than item2. - inline bool is_more_right(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) - { - if (inserted_right(item2)) + // Helper function which returns true if the item was inserted at the right. + inline bool inserted_right(Item *item) { - if (inserted_right(item1)) + return item->index.load() > 0; + } + + // Helper function which returns true if item1 is more left than item2. + inline bool is_more_left(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) + { + if (inserted_left(item2)) { - return timestamping_->is_later(timestamp1, timestamp2); + if (inserted_left(item1)) + { + return timestamping_->is_later(timestamp1, timestamp2); + } + else + { + return false; + } } else { - return false; + if (inserted_left(item1)) + { + return true; + } + else + { + return timestamping_->is_later(timestamp2, timestamp1); + } } } - else + + // Helper function which returns true if item1 is more right than item2. + inline bool is_more_right(Item *item1, uint64_t *timestamp1, Item *item2, uint64_t *timestamp2) { - if (inserted_right(item1)) + if (inserted_right(item2)) { - return true; + if (inserted_right(item1)) + { + return timestamping_->is_later(timestamp1, timestamp2); + } + else + { + return false; + } } else { - return timestamping_->is_later(timestamp2, timestamp1); + if (inserted_right(item1)) + { + return true; + } + else + { + return timestamping_->is_later(timestamp2, timestamp1); + } } } - } - bool try_remove_left(T *element, uint64_t *invocation_time) - { - // Initialize the data needed for the emptiness check. - uint64_t thread_id = ThreadContext::get().thread_id(); - Item **emptiness_check_left = - emptiness_check_left_[thread_id]; - Item **emptiness_check_right = - emptiness_check_right_[thread_id]; - bool empty = true; - // Initialize the result pointer to NULL, which means that no - // element has been removed. - Item *result = NULL; - // Indicates the index which contains the youngest item. - uint64_t buffer_index = -1; - // Memory on the stack frame where timestamps of items can be stored - // temporarily. - uint64_t tmp_timestamp[2][2]; - // Index in the tmp_timestamp array which is not used at the moment. - uint64_t tmp_index = 1; - timestamping_->init_sentinel(tmp_timestamp[0]); - uint64_t *timestamp = tmp_timestamp[0]; - // Stores the value of the remove pointer of a thead-local buffer - // before the buffer is actually accessed. - Item *old_left = NULL; - - // Read the start time of the iteration. Items which were timestamped - // after the start time and inserted at the right are not removed. - uint64_t start_time[2]; - timestamping_->read_time(start_time); - // We start iterating over the thread-local lists at a random index. - uint64_t start = hwrand(); - // We iterate over all thead-local buffers - for (uint64_t i = 0; i < num_threads_; i++) + bool try_remove_left(T *element, uint64_t *invocation_time) { - - uint64_t tmp_buffer_index = (start + i) % num_threads_; - // We get the remove/insert pointer of the current thread-local buffer. - Item *tmp_left = left_[tmp_buffer_index]->load(); - // We get the youngest element from that thread-local buffer. - Item *item = get_left_item(tmp_buffer_index); - // If we found an element, we compare it to the youngest element - // we have found until now. - if (item != NULL) + // Initialize the data needed for the emptiness check. + uint64_t thread_id = get_thread_id(); + Item **emptiness_check_left = + emptiness_check_left_[thread_id]; + Item **emptiness_check_right = + emptiness_check_right_[thread_id]; + bool empty = true; + // Initialize the result pointer to NULL, which means that no + // element has been removed. + Item *result = NULL; + // Indicates the index which contains the youngest item. + uint64_t buffer_index = -1; + // Memory on the stack frame where timestamps of items can be stored + // temporarily. + uint64_t tmp_timestamp[2][2]; + // Index in the tmp_timestamp array which is not used at the moment. + uint64_t tmp_index = 1; + timestamping_->init_sentinel(tmp_timestamp[0]); + uint64_t *timestamp = tmp_timestamp[0]; + // Stores the value of the remove pointer of a thead-local buffer + // before the buffer is actually accessed. + Item *old_left = NULL; + + // Read the start time of the iteration. Items which were timestamped + // after the start time and inserted at the right are not removed. + uint64_t start_time[2]; + timestamping_->read_time(start_time); + // We start iterating over the thread-local lists at a random index. + uint64_t start = random_engine_(); + // We iterate over all thead-local buffers + for (uint64_t i = 0; i < num_threads_; i++) { - empty = false; - uint64_t *item_timestamp; - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; - - if (inserted_left(item) && !timestamping_->is_later(invocation_time, item_timestamp)) + uint64_t tmp_buffer_index = (start + i) % num_threads_; + // We get the remove/insert pointer of the current thread-local buffer. + Item *tmp_left = left_[tmp_buffer_index]->load(); + // We get the youngest element from that thread-local buffer. + Item *item = get_left_item(tmp_buffer_index); + // If we found an element, we compare it to the youngest element + // we have found until now. + if (item != NULL) { - uint64_t expected = 0; - if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) - { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - left_[tmp_buffer_index]->compare_exchange_weak( - tmp_left, (Item *)add_next_aba(item, tmp_left, 0)); - *element = item->data.load(); - return true; - } - else + empty = false; + uint64_t *item_timestamp; + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + + if (inserted_left(item) && !timestamping_->is_later(invocation_time, item_timestamp)) { - item = get_left_item(tmp_buffer_index); - if (item != NULL) + uint64_t expected = 0; + if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) { - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + left_[tmp_buffer_index]->compare_exchange_weak( + tmp_left, (Item *)add_next_aba(item, tmp_left, 0)); + *element = item->data.load(); + return true; + } + else + { + item = get_left_item(tmp_buffer_index); + if (item != NULL) + { + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + } } } - } - if (item != NULL && (result == NULL || is_more_left(item, item_timestamp, result, timestamp))) - { - // We found a new leftmost item, so we remember it. - result = item; - buffer_index = tmp_buffer_index; - timestamp = item_timestamp; - tmp_index ^= 1; - old_left = tmp_left; - - // Check if we can remove the element immediately. - if (inserted_left(result) && !timestamping_->is_later(invocation_time, timestamp)) + if (item != NULL && (result == NULL || is_more_left(item, item_timestamp, result, timestamp))) { - uint64_t expected = 0; - if (result->taken.load() == 0) + // We found a new leftmost item, so we remember it. + result = item; + buffer_index = tmp_buffer_index; + timestamp = item_timestamp; + tmp_index ^= 1; + old_left = tmp_left; + + // Check if we can remove the element immediately. + if (inserted_left(result) && !timestamping_->is_later(invocation_time, timestamp)) { - if (result->taken.compare_exchange_weak( - expected, 1)) + uint64_t expected = 0; + if (result->taken.load() == 0) { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - left_[buffer_index]->compare_exchange_weak( - old_left, (Item *)add_next_aba(result, old_left, 0)); - - *element = result->data.load(); - return true; + if (result->taken.compare_exchange_weak( + expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + left_[buffer_index]->compare_exchange_weak( + old_left, (Item *)add_next_aba(result, old_left, 0)); + + *element = result->data.load(); + return true; + } } } } } - } - else - { - // No element was found, work on the emptiness check. - if (emptiness_check_left[tmp_buffer_index] != tmp_left) - { - empty = false; - emptiness_check_left[tmp_buffer_index] = - tmp_left; - } - Item *tmp_right = right_[tmp_buffer_index]->load(); - if (emptiness_check_right[tmp_buffer_index] != tmp_right) + else { - empty = false; - emptiness_check_right[tmp_buffer_index] = - tmp_right; + // No element was found, work on the emptiness check. + if (emptiness_check_left[tmp_buffer_index] != tmp_left) + { + empty = false; + emptiness_check_left[tmp_buffer_index] = + tmp_left; + } + Item *tmp_right = right_[tmp_buffer_index]->load(); + if (emptiness_check_right[tmp_buffer_index] != tmp_right) + { + empty = false; + emptiness_check_right[tmp_buffer_index] = + tmp_right; + } } } - } - if (result != NULL) - { - if (!timestamping_->is_later(timestamp, start_time)) + if (result != NULL) { - // The found item was timestamped after the start of the iteration, - // so it is save to remove it. - uint64_t expected = 0; - if (result->taken.load() == 0) + if (!timestamping_->is_later(timestamp, start_time)) { - if (result->taken.compare_exchange_weak( - expected, 1)) + // The found item was timestamped after the start of the iteration, + // so it is save to remove it. + uint64_t expected = 0; + if (result->taken.load() == 0) { - // Try to adjust the remove pointer. It does not matter if this - // CAS fails. - left_[buffer_index]->compare_exchange_weak( - old_left, (Item *)add_next_aba(result, old_left, 0)); - *element = result->data.load(); - return true; + if (result->taken.compare_exchange_weak( + expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if this + // CAS fails. + left_[buffer_index]->compare_exchange_weak( + old_left, (Item *)add_next_aba(result, old_left, 0)); + *element = result->data.load(); + return true; + } } } } - } - *element = (T)NULL; - return !empty; - } + element = NULL; + return !empty; + } - bool try_remove_right(T *element, uint64_t *invocation_time) - { - // Initialize the data needed for the emptiness check. - uint64_t thread_id = ThreadContext::get().thread_id(); - Item **emptiness_check_left = - emptiness_check_left_[thread_id]; - Item **emptiness_check_right = - emptiness_check_right_[thread_id]; - bool empty = true; - // Initialize the result pointer to NULL, which means that no - // element has been removed. - Item *result = NULL; - // Indicates the index which contains the youngest item. - uint64_t buffer_index = -1; - // Memory on the stack frame where timestamps of items can be stored - // temporarily. - uint64_t tmp_timestamp[2][2]; - // Index in the tmp_timestamp array whihc is not used at the moment. - uint64_t tmp_index = 1; - timestamping_->init_sentinel(tmp_timestamp[0]); - uint64_t *timestamp = tmp_timestamp[0]; - // Stores the value of the remove pointer of a thead-local buffer - // before the buffer is actually accessed. - Item *old_right = NULL; - - // Read the start time of the iteration. Items which were timestamped - // after the start time and inserted at the left are not removed. - uint64_t start_time[2]; - timestamping_->read_time(start_time); - // We start iterating over the thread-local lists at a random index. - uint64_t start = hwrand(); - // We iterate over all thead-local buffers - for (uint64_t i = 0; i < num_threads_; i++) + bool try_remove_right(T *element, uint64_t *invocation_time) { - - uint64_t tmp_buffer_index = (start + i) % num_threads_; - // We get the remove/insert pointer of the current thread-local buffer. - Item *tmp_right = right_[tmp_buffer_index]->load(); - // We get the youngest element from that thread-local buffer. - Item *item = get_right_item(tmp_buffer_index); - // If we found an element, we compare it to the youngest element - // we have found until now. - if (item != NULL) + // Initialize the data needed for the emptiness check. + uint64_t thread_id = get_thread_id(); + Item **emptiness_check_left = + emptiness_check_left_[thread_id]; + Item **emptiness_check_right = + emptiness_check_right_[thread_id]; + bool empty = true; + // Initialize the result pointer to NULL, which means that no + // element has been removed. + Item *result = NULL; + // Indicates the index which contains the youngest item. + uint64_t buffer_index = -1; + // Memory on the stack frame where timestamps of items can be stored + // temporarily. + uint64_t tmp_timestamp[2][2]; + // Index in the tmp_timestamp array whihc is not used at the moment. + uint64_t tmp_index = 1; + timestamping_->init_sentinel(tmp_timestamp[0]); + uint64_t *timestamp = tmp_timestamp[0]; + // Stores the value of the remove pointer of a thead-local buffer + // before the buffer is actually accessed. + Item *old_right = NULL; + + // Read the start time of the iteration. Items which were timestamped + // after the start time and inserted at the left are not removed. + uint64_t start_time[2]; + timestamping_->read_time(start_time); + // We start iterating over the thread-local lists at a random index. + uint64_t start = random_engine_(); + // We iterate over all thead-local buffers + for (uint64_t i = 0; i < num_threads_; i++) { - empty = false; - uint64_t *item_timestamp; - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; - - if (inserted_right(item) && !timestamping_->is_later(invocation_time, item_timestamp)) + uint64_t tmp_buffer_index = (start + i) % num_threads_; + // We get the remove/insert pointer of the current thread-local buffer. + Item *tmp_right = right_[tmp_buffer_index]->load(); + // We get the youngest element from that thread-local buffer. + Item *item = get_right_item(tmp_buffer_index); + // If we found an element, we compare it to the youngest element + // we have found until now. + if (item != NULL) { - uint64_t expected = 0; - if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) - { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - right_[tmp_buffer_index]->compare_exchange_weak( - tmp_right, (Item *)add_next_aba(item, tmp_right, 0)); - *element = item->data.load(); - return true; - } - else + empty = false; + uint64_t *item_timestamp; + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + + if (inserted_right(item) && !timestamping_->is_later(invocation_time, item_timestamp)) { - item = get_right_item(tmp_buffer_index); - if (item != NULL) + uint64_t expected = 0; + if (item->taken.load() == 0 && item->taken.compare_exchange_weak(expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + right_[tmp_buffer_index]->compare_exchange_weak( + tmp_right, (Item *)add_next_aba(item, tmp_right, 0)); + *element = item->data.load(); + return true; + } + else { - timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); - item_timestamp = tmp_timestamp[tmp_index]; + item = get_right_item(tmp_buffer_index); + if (item != NULL) + { + timestamping_->load_timestamp(tmp_timestamp[tmp_index], item->timestamp); + item_timestamp = tmp_timestamp[tmp_index]; + } } } - } - if (item != NULL && (result == NULL || is_more_right(item, item_timestamp, result, timestamp))) - { - // We found a new youngest element, so we remember it. - result = item; - buffer_index = tmp_buffer_index; - timestamp = item_timestamp; - tmp_index ^= 1; - old_right = tmp_right; - } - } - else - { - // No element was found, work on the emptiness check. - if (emptiness_check_right[tmp_buffer_index] != tmp_right) - { - empty = false; - emptiness_check_right[tmp_buffer_index] = - tmp_right; + if (item != NULL && (result == NULL || is_more_right(item, item_timestamp, result, timestamp))) + { + // We found a new youngest element, so we remember it. + result = item; + buffer_index = tmp_buffer_index; + timestamp = item_timestamp; + tmp_index ^= 1; + old_right = tmp_right; + } } - Item *tmp_left = left_[tmp_buffer_index]->load(); - if (emptiness_check_left[tmp_buffer_index] != tmp_left) + else { - empty = false; - emptiness_check_left[tmp_buffer_index] = - tmp_left; + // No element was found, work on the emptiness check. + if (emptiness_check_right[tmp_buffer_index] != tmp_right) + { + empty = false; + emptiness_check_right[tmp_buffer_index] = + tmp_right; + } + Item *tmp_left = left_[tmp_buffer_index]->load(); + if (emptiness_check_left[tmp_buffer_index] != tmp_left) + { + empty = false; + emptiness_check_left[tmp_buffer_index] = + tmp_left; + } } } - } - if (result != NULL) - { - if (!timestamping_->is_later(timestamp, start_time)) + if (result != NULL) { - // The found item was timestamped after the start of the iteration, - // so it is save to remove it. - uint64_t expected = 0; - if (result->taken.load() == 0) + if (!timestamping_->is_later(timestamp, start_time)) { - if (result->taken.compare_exchange_weak( - expected, 1)) + // The found item was timestamped after the start of the iteration, + // so it is save to remove it. + uint64_t expected = 0; + if (result->taken.load() == 0) { - // Try to adjust the remove pointer. It does not matter if - // this CAS fails. - right_[buffer_index]->compare_exchange_weak( - old_right, (Item *)add_next_aba(result, old_right, 0)); - *element = result->data.load(); - return true; + if (result->taken.compare_exchange_weak( + expected, 1)) + { + // Try to adjust the remove pointer. It does not matter if + // this CAS fails. + right_[buffer_index]->compare_exchange_weak( + old_right, (Item *)add_next_aba(result, old_right, 0)); + *element = result->data.load(); + return true; + } } } } + + element = NULL; + return !empty; } + }; + +}} // namespace cds::container - *element = (T)NULL; - return !empty; - } -}; +#endif // #ifndef CDSLIB_CONTAINER_TS_DEQUE_BUFFER_H diff --git a/cds/container/ts_timestamp.h b/cds/container/ts_timestamp.h index 3c7665e22..17c15578c 100644 --- a/cds/container/ts_timestamp.h +++ b/cds/container/ts_timestamp.h @@ -1,185 +1,194 @@ +#ifndef CDSLIB_CONTAINER_TS_TIMESTAMP_H +#define CDSLIB_CONTAINER_TS_TIMESTAMP_H + #include -inline uint64_t get_hwptime(void) -{ - uint64_t aux; - uint64_t rax, rdx; - asm volatile ("rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); - return (rdx << 32) + rax; -} - -inline uint64_t get_hwtime(void) -{ - unsigned int hi, lo; - __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); - return ((uint64_t)lo) | (((uint64_t)hi) << 32); -} - -class HardwareTimestamp -{ -public: - inline void initialize(uint64_t delay, uint64_t num_threads) - { - } +namespace cds { namespace container { - inline void init_sentinel(uint64_t *result) + inline uint64_t get_hwptime(void) { - result[0] = 0; + uint64_t aux; + uint64_t rax, rdx; + asm volatile ("rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); + return (rdx << 32) + rax; } - inline void init_sentinel_atomic(std::atomic *result) + inline uint64_t get_hwtime(void) { - result[0].store(0); + unsigned int hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t)lo) | (((uint64_t)hi) << 32); } - inline void init_top_atomic(std::atomic *result) + class HardwareTimestamp { - result[0].store(UINT64_MAX); - } + public: + inline void initialize(uint64_t delay, uint64_t num_threads) + { + } - inline void init_top(uint64_t *result) - { - result[0] = UINT64_MAX; - } + inline void init_sentinel(uint64_t *result) + { + result[0] = 0; + } - inline void load_timestamp(uint64_t *result, std::atomic *source) - { - result[0] = source[0].load(); - } + inline void init_sentinel_atomic(std::atomic *result) + { + result[0].store(0); + } - inline void set_timestamp(std::atomic *result) - { - result[0].store(get_hwptime()); - } + inline void init_top_atomic(std::atomic *result) + { + result[0].store(UINT64_MAX); + } - inline void read_time(uint64_t *result) - { - result[0] = get_hwptime(); - } + inline void init_top(uint64_t *result) + { + result[0] = UINT64_MAX; + } - inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) - { - return timestamp2[0] < timestamp1[0]; - } -}; + inline void load_timestamp(uint64_t *result, std::atomic *source) + { + result[0] = source[0].load(); + } -class HardwareIntervalTimestamp -{ -private: - uint64_t delay_; + inline void set_timestamp(std::atomic *result) + { + result[0].store(get_hwptime()); + } -public: - inline void initialize(uint64_t delay, uint64_t num_threads) - { - delay_ = delay; - } + inline void read_time(uint64_t *result) + { + result[0] = get_hwptime(); + } - inline void init_sentinel(uint64_t *result) - { - result[0] = 0; - result[1] = 0; - } + inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) + { + return timestamp2[0] < timestamp1[0]; + } + }; - inline void init_sentinel_atomic(std::atomic *result) + class HardwareIntervalTimestamp { - result[0].store(0); - result[1].store(0); - } + private: + uint64_t delay_; - inline void init_top_atomic(std::atomic *result) - { - result[0].store(UINT64_MAX); - result[1].store(UINT64_MAX); - } + public: + inline void initialize(uint64_t delay, uint64_t num_threads) + { + delay_ = delay; + } - inline void init_top(uint64_t *result) - { - result[0] = UINT64_MAX; - result[1] = UINT64_MAX; - } + inline void init_sentinel(uint64_t *result) + { + result[0] = 0; + result[1] = 0; + } - inline void load_timestamp(uint64_t *result, std::atomic *source) - { - result[0] = source[0].load(); - result[1] = source[1].load(); - } + inline void init_sentinel_atomic(std::atomic *result) + { + result[0].store(0); + result[1].store(0); + } - inline void set_timestamp(std::atomic *result) - { - result[0].store(get_hwptime()); - uint64_t wait = get_hwtime() + delay_; - while (get_hwtime() < wait) + inline void init_top_atomic(std::atomic *result) { + result[0].store(UINT64_MAX); + result[1].store(UINT64_MAX); } - result[1].store(get_hwptime()); - } - inline void read_time(uint64_t *result) - { - result[0] = get_hwptime(); - result[1] = result[0]; - } + inline void init_top(uint64_t *result) + { + result[0] = UINT64_MAX; + result[1] = UINT64_MAX; + } - inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) - { - return timestamp2[1] < timestamp1[0]; - } -}; + inline void load_timestamp(uint64_t *result, std::atomic *source) + { + result[0] = source[0].load(); + result[1] = source[1].load(); + } -class AtomicCounterTimestamp -{ -private: - std::atomic *clock_; + inline void set_timestamp(std::atomic *result) + { + result[0].store(get_hwptime()); + uint64_t wait = get_hwtime() + delay_; + while (get_hwtime() < wait) + { + } + result[1].store(get_hwptime()); + } -public: - inline void initialize(uint64_t delay, uint64_t num_threads) - { - clock_ = new std::atomic(); - clock_->store(1); - } + inline void read_time(uint64_t *result) + { + result[0] = get_hwptime(); + result[1] = result[0]; + } - inline void init_sentinel(uint64_t *result) - { - result[0] = 0; - } + inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) + { + return timestamp2[1] < timestamp1[0]; + } + }; - inline void init_sentinel_atomic(std::atomic *result) + class AtomicCounterTimestamp { - result[0].store(0); - } + private: + std::atomic *clock_; - inline void init_top_atomic(std::atomic *result) - { - result[0].store(UINT64_MAX); - } + public: + inline void initialize(uint64_t delay, uint64_t num_threads) + { + clock_ = new std::atomic(); + clock_->store(1); + } - inline void init_top(uint64_t *result) - { - result[0] = UINT64_MAX; - } + inline void init_sentinel(uint64_t *result) + { + result[0] = 0; + } - inline void load_timestamp(uint64_t *result, std::atomic *source) - { - result[0] = source[0].load(); - } + inline void init_sentinel_atomic(std::atomic *result) + { + result[0].store(0); + } - inline void set_timestamp(std::atomic *result) - { - result[0].store(clock_->fetch_add(1)); - } + inline void init_top_atomic(std::atomic *result) + { + result[0].store(UINT64_MAX); + } - inline void set_timestamp_local(uint64_t *result) - { - result[0] = clock_->fetch_add(1); - } + inline void init_top(uint64_t *result) + { + result[0] = UINT64_MAX; + } - inline void read_time(uint64_t *result) - { - result[0] = clock_->load(); - } + inline void load_timestamp(uint64_t *result, std::atomic *source) + { + result[0] = source[0].load(); + } - inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) - { - return timestamp2[0] < timestamp1[0]; - } -}; + inline void set_timestamp(std::atomic *result) + { + result[0].store(clock_->fetch_add(1)); + } + + inline void set_timestamp_local(uint64_t *result) + { + result[0] = clock_->fetch_add(1); + } + + inline void read_time(uint64_t *result) + { + result[0] = clock_->load(); + } + + inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) + { + return timestamp2[0] < timestamp1[0]; + } + }; + +}} // namespace cds::container + +#endif // #ifndef CDSLIB_CONTAINER_TS_TIMESTAMP_H From eb5e9f6ed5f5e03089304cb304b6da022bb36769 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Mon, 24 Dec 2018 23:52:25 +0300 Subject: [PATCH 21/29] add unit tests --- test/unit/deque/CMakeLists.txt | 3 +- test/unit/deque/ts_deque.cpp | 116 +++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 test/unit/deque/ts_deque.cpp diff --git a/test/unit/deque/CMakeLists.txt b/test/unit/deque/CMakeLists.txt index cf8fb8076..6985edff5 100644 --- a/test/unit/deque/CMakeLists.txt +++ b/test/unit/deque/CMakeLists.txt @@ -3,6 +3,7 @@ set(PACKAGE_NAME unit-deque) set(CDSGTEST_DEQUE_SOURCES ../main.cpp fcdeque.cpp + ts_deque.cpp ) include_directories( @@ -12,4 +13,4 @@ include_directories( add_executable(${PACKAGE_NAME} ${CDSGTEST_DEQUE_SOURCES}) target_link_libraries(${PACKAGE_NAME} ${CDS_TEST_LIBRARIES}) -add_test(NAME ${PACKAGE_NAME} COMMAND ${PACKAGE_NAME} WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) \ No newline at end of file +add_test(NAME ${PACKAGE_NAME} COMMAND ${PACKAGE_NAME} WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) diff --git a/test/unit/deque/ts_deque.cpp b/test/unit/deque/ts_deque.cpp new file mode 100644 index 000000000..808efa5c7 --- /dev/null +++ b/test/unit/deque/ts_deque.cpp @@ -0,0 +1,116 @@ +#include +#include +#include + +namespace { + + class TSDeque: public ::testing::Test + { + protected: + template + void test( Deque& dq ) + { + size_t const c_nSize = 100; + + // insert_right/remove_right + for ( int i = 0; i < static_cast( c_nSize ); ++i ) { + EXPECT_TRUE( dq.insert_right( i )); + } + EXPECT_EQ( dq.size(), c_nSize ); + + size_t nCount = 0; + int val; + while ( !dq.empty()) { + EXPECT_TRUE( dq.remove_right( &val )); + ++nCount; + EXPECT_EQ( static_cast(c_nSize - nCount), val ); + } + EXPECT_EQ( nCount, c_nSize ); + + // insert_left/remove_left + for ( int i = 0; i < static_cast( c_nSize ); ++i ) + EXPECT_TRUE( dq.insert_left( i )); + EXPECT_EQ( dq.size(), c_nSize ); + + nCount = 0; + while ( !dq.empty()) { + EXPECT_TRUE( dq.remove_left( &val )); + ++nCount; + EXPECT_EQ( static_cast(c_nSize - nCount), val ); + } + EXPECT_EQ( nCount, c_nSize ); + + // insert_left/remove_right + for ( int i = 0; i < static_cast( c_nSize ); ++i ) + EXPECT_TRUE( dq.insert_left( i )); + EXPECT_EQ( dq.size(), c_nSize ); + + nCount = 0; + while ( !dq.empty()) { + EXPECT_TRUE( dq.remove_right( &val )); + EXPECT_EQ( static_cast( nCount ), val ); + ++nCount; + } + EXPECT_EQ( nCount, c_nSize ); + + // insert_right/remove_left + for ( int i = 0; i < static_cast( c_nSize ); ++i ) + EXPECT_TRUE( dq.insert_right( i )); + EXPECT_EQ( dq.size(), c_nSize ); + + nCount = 0; + while ( !dq.empty()) { + EXPECT_TRUE( dq.remove_left( &val )); + EXPECT_EQ( static_cast( nCount ), val ); + ++nCount; + } + EXPECT_EQ( nCount, c_nSize ); + + // clear + for ( int i = 0; i < static_cast( c_nSize ); ++i ) + EXPECT_TRUE( dq.insert_right( i )); + EXPECT_EQ( dq.size(), c_nSize ); + + EXPECT_FALSE( dq.empty()); + dq.clear(); + EXPECT_TRUE( dq.empty()); + } + }; + + TEST_F( TSDeque, hardware_timestamping ) + { + typedef cds::container::TSDeque + >::type + > deque_type; + + deque_type dq(1, 0); + test( dq ); + } + + TEST_F( TSDeque, hardware_interval_timestamping ) + { + typedef cds::container::TSDeque + >::type + > deque_type; + + deque_type dq(1, 1000); + test( dq ); + } + + TEST_F( TSDeque, atomic_counter_timestamping ) + { + typedef cds::container::TSDeque + >::type + > deque_type; + + deque_type dq(1, 0); + test( dq ); + } + +} // namespace From a5ecc161d95cc829f52f0f562f8dbfbd96b9fa38 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Tue, 25 Dec 2018 00:17:34 +0300 Subject: [PATCH 22/29] move asm code to compiler folder --- cds/compiler/gcc/amd64/ts_hardwaretimestamp.h | 33 +++++++++++++++++++ cds/compiler/gcc/x86/ts_hardwaretimestamp.h | 33 +++++++++++++++++++ cds/compiler/ts_hardwaretimestamp.h | 16 +++++++++ cds/container/ts_deque_buffer.h | 1 + cds/container/ts_timestamp.h | 30 +++++------------ 5 files changed, 92 insertions(+), 21 deletions(-) create mode 100644 cds/compiler/gcc/amd64/ts_hardwaretimestamp.h create mode 100644 cds/compiler/gcc/x86/ts_hardwaretimestamp.h create mode 100644 cds/compiler/ts_hardwaretimestamp.h diff --git a/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h b/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h new file mode 100644 index 000000000..7cbc40709 --- /dev/null +++ b/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h @@ -0,0 +1,33 @@ +#ifndef CDSLIB_COMPILER_GCC_AMD64_TS_HARDWARETIMESTAMP_H +#define CDSLIB_COMPILER_GCC_AMD64_TS_HARDWARETIMESTAMP_H + +#include + +namespace cds { namespace tshardwaretimestamp { + namespace gcc { namespace amd64 { + +# define CDS_ts_hardwaretimestamp_hwptime_defined + static inline uint64_t get_hwptime() + { + uint64_t aux; + uint64_t rax, rdx; + asm volatile ("rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); + return (rdx << 32) + rax; + } + +# define CDS_ts_hardwaretimestamp_hwtime_defined + static inline uint64_t get_hwtime() + { + unsigned int hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t)lo) | (((uint64_t)hi) << 32); + } + + }} // namespace gcc::amd64 + + namespace platform { + using namespace gcc::amd64; + } +}} // namespace cds::tshardwaretimestamp + +#endif // #ifndef CDSLIB_COMPILER_GCC_AMD64_TS_HARDWARETIMESTAMP_H diff --git a/cds/compiler/gcc/x86/ts_hardwaretimestamp.h b/cds/compiler/gcc/x86/ts_hardwaretimestamp.h new file mode 100644 index 000000000..1e14dfcc5 --- /dev/null +++ b/cds/compiler/gcc/x86/ts_hardwaretimestamp.h @@ -0,0 +1,33 @@ +#ifndef CDSLIB_COMPILER_GCC_X86_TS_HARDWARETIMESTAMP_H +#define CDSLIB_COMPILER_GCC_X86_TS_HARDWARETIMESTAMP_H + +#include + +namespace cds { namespace tshardwaretimestamp { + namespace gcc { namespace x86 { + +# define CDS_ts_hardwaretimestamp_hwptime_defined + static inline uint64_t get_hwptime() + { + uint64_t aux; + uint64_t rax, rdx; + asm volatile ("rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); + return (rdx << 32) + rax; + } + +# define CDS_ts_hardwaretimestamp_hwtime_defined + static inline uint64_t get_hwtime() + { + unsigned int hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t)lo) | (((uint64_t)hi) << 32); + } + + }} // namespace gcc::x86 + + namespace platform { + using namespace gcc::x86; + } +}} // namespace cds::tshardwaretimestamp + +#endif // #ifndef CDSLIB_COMPILER_GCC_X86_TS_HARDWARETIMESTAMP_H diff --git a/cds/compiler/ts_hardwaretimestamp.h b/cds/compiler/ts_hardwaretimestamp.h new file mode 100644 index 000000000..0a4352b77 --- /dev/null +++ b/cds/compiler/ts_hardwaretimestamp.h @@ -0,0 +1,16 @@ +#ifndef CDSLIB_COMPILER_TS_HARDWARETIMESTAMP_H +#define CDSLIB_COMPILER_TS_HARDWARETIMESTAMP_H + +// Choose appropriate header for current architecture and compiler + +#if CDS_COMPILER == CDS_COMPILER_GCC || CDS_COMPILER == CDS_COMPILER_CLANG || CDS_COMPILER == CDS_COMPILER_INTEL +# if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86 +# include +# elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64 +# include +# endif +#else +# error "Undefined compiler" +#endif + +#endif // #ifndef CDSLIB_COMPILER_TS_HARDWARETIMESTAMP_H diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index b1d74e6ab..fc654a90e 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -40,6 +40,7 @@ namespace cds { namespace container { std::atomic thread_id_counter_; boost::thread_specific_ptr thread_id_; + // Helper function to get index of thread int get_thread_id() { int* temp = thread_id_.get(); diff --git a/cds/container/ts_timestamp.h b/cds/container/ts_timestamp.h index 17c15578c..042a6e265 100644 --- a/cds/container/ts_timestamp.h +++ b/cds/container/ts_timestamp.h @@ -2,23 +2,11 @@ #define CDSLIB_CONTAINER_TS_TIMESTAMP_H #include +#include namespace cds { namespace container { - inline uint64_t get_hwptime(void) - { - uint64_t aux; - uint64_t rax, rdx; - asm volatile ("rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); - return (rdx << 32) + rax; - } - - inline uint64_t get_hwtime(void) - { - unsigned int hi, lo; - __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); - return ((uint64_t)lo) | (((uint64_t)hi) << 32); - } + using namespace cds::tshardwaretimestamp; class HardwareTimestamp { @@ -54,12 +42,12 @@ namespace cds { namespace container { inline void set_timestamp(std::atomic *result) { - result[0].store(get_hwptime()); + result[0].store(platform::get_hwptime()); } inline void read_time(uint64_t *result) { - result[0] = get_hwptime(); + result[0] = platform::get_hwptime(); } inline bool is_later(uint64_t *timestamp1, uint64_t *timestamp2) @@ -111,17 +99,17 @@ namespace cds { namespace container { inline void set_timestamp(std::atomic *result) { - result[0].store(get_hwptime()); - uint64_t wait = get_hwtime() + delay_; - while (get_hwtime() < wait) + result[0].store(platform::get_hwptime()); + uint64_t wait = platform::get_hwtime() + delay_; + while (platform::get_hwtime() < wait) { } - result[1].store(get_hwptime()); + result[1].store(platform::get_hwptime()); } inline void read_time(uint64_t *result) { - result[0] = get_hwptime(); + result[0] = platform::get_hwptime(); result[1] = result[0]; } From 72f5ef0251b068da015e0469d91d58498567687a Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Tue, 25 Dec 2018 00:37:48 +0300 Subject: [PATCH 23/29] fix --- cds/container/ts_deque.h | 3 +-- cds/container/ts_deque_buffer.h | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/cds/container/ts_deque.h b/cds/container/ts_deque.h index 5ac7722a1..f174a167b 100644 --- a/cds/container/ts_deque.h +++ b/cds/container/ts_deque.h @@ -100,8 +100,7 @@ namespace cds { namespace container { timestamping_ = new timestamp(); timestamping_->initialize(delay, num_threads); - buffer_ = new TSDequeBuffer(); - buffer_->initialize(num_threads, timestamping_); + buffer_ = new TSDequeBuffer(num_threads, timestamping_); } ~TSDeque() diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index fc654a90e..11a249907 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -161,7 +161,7 @@ namespace cds { namespace container { } public: - void initialize(uint64_t num_threads, TimeStamp *timestamping) + TSDequeBuffer(uint64_t num_threads, TimeStamp *timestamping) { thread_id_counter_.store(0); @@ -200,6 +200,24 @@ namespace cds { namespace container { } } + ~TSDequeBuffer() + { + for (uint64_t i = 0; i < num_threads_; i++) + { + delete left_[i]; + delete right_[i]; + delete next_index_[i]; + delete[] emptiness_check_left_[i]; + delete[] emptiness_check_right_[i]; + } + delete[] left_; + delete[] right_; + delete[] next_index_; + delete[] emptiness_check_left_; + delete[] emptiness_check_right_; + + } + inline std::atomic *insert_left(T element) { uint64_t thread_id = get_thread_id(); From ac02558f8c7eed896dd49840408aa28a59fac8ae Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Tue, 25 Dec 2018 20:53:43 +0300 Subject: [PATCH 24/29] add copyright note --- cds/container/ts_deque.h | 12 ------------ cds/container/ts_deque_buffer.h | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/cds/container/ts_deque.h b/cds/container/ts_deque.h index f174a167b..e0b2c9162 100644 --- a/cds/container/ts_deque.h +++ b/cds/container/ts_deque.h @@ -118,9 +118,6 @@ namespace cds { namespace container { bool insert_left(value_type element) { std::atomic *item = buffer_->insert_left(element); - // In the set_timestamp operation first a new timestamp is acquired - // and then assigned to the item. The operation may not be executed - // atomically. timestamping_->set_timestamp(item); ++item_counter_; return true; @@ -133,9 +130,6 @@ namespace cds { namespace container { bool insert_right(value_type element) { std::atomic *item = buffer_->insert_right(element); - // In the set_timestamp operation first a new timestamp is acquired - // and then assigned to the item. The operation may not be executed - // atomically. timestamping_->set_timestamp(item); ++item_counter_; return true; @@ -147,8 +141,6 @@ namespace cds { namespace container { */ bool remove_left(value_type *element) { - // Read the invocation time of this operation, needed for the - // elimination optimization. uint64_t invocation_time[2]; timestamping_->read_time(invocation_time); while (buffer_->try_remove_left(element, invocation_time)) @@ -159,7 +151,6 @@ namespace cds { namespace container { return true; } } - // The deque was empty, return false. return false; } @@ -169,8 +160,6 @@ namespace cds { namespace container { */ bool remove_right(value_type *element) { - // Read the invocation time of this operation, needed for the - // elimination optimization. uint64_t invocation_time[2]; timestamping_->read_time(invocation_time); while (buffer_->try_remove_right(element, invocation_time)) @@ -181,7 +170,6 @@ namespace cds { namespace container { return true; } } - // The deque was empty, return false. return false; } diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index 11a249907..dcd43bc95 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -1,3 +1,22 @@ +/* +Copyright (c) 2012-2016, the Scal Project Authors. All rights reserved. Please see the AUTHORS file for details. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the Scal Project. + +-------------------------------------------------- + +Modified by: + Alexey Chirukhin https://github.com/pr3sto + Eduard Blees https://github.com/EduardBlees +*/ + #ifndef CDSLIB_CONTAINER_TS_DEQUE_BUFFER_H #define CDSLIB_CONTAINER_TS_DEQUE_BUFFER_H From 430297f782dea16a92120e47cab9e699efec0b4f Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Fri, 28 Dec 2018 02:54:00 +0300 Subject: [PATCH 25/29] fix emptyness check, thread_id --- cds/container/ts_deque.h | 18 ++++++++++++++---- cds/container/ts_deque_buffer.h | 31 ++++++++++++------------------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/cds/container/ts_deque.h b/cds/container/ts_deque.h index e0b2c9162..7352e38e7 100644 --- a/cds/container/ts_deque.h +++ b/cds/container/ts_deque.h @@ -143,9 +143,10 @@ namespace cds { namespace container { { uint64_t invocation_time[2]; timestamping_->read_time(invocation_time); - while (buffer_->try_remove_left(element, invocation_time)) + bool empty; + while (buffer_->try_remove_left(element, invocation_time, &empty)) { - if (element != NULL) + if (!empty) { --item_counter_; return true; @@ -162,9 +163,10 @@ namespace cds { namespace container { { uint64_t invocation_time[2]; timestamping_->read_time(invocation_time); - while (buffer_->try_remove_right(element, invocation_time)) + bool empty; + while (buffer_->try_remove_right(element, invocation_time, &empty)) { - if (element != NULL) + if (!empty) { --item_counter_; return true; @@ -208,6 +210,14 @@ namespace cds { namespace container { { return item_counter_; } + + //@cond + /// The class has no internal statistics. For test consistency only + std::nullptr_t statistics() const + { + return nullptr; + } + //@endcond }; }} // namespace cds::container diff --git a/cds/container/ts_deque_buffer.h b/cds/container/ts_deque_buffer.h index dcd43bc95..c1e8ed466 100644 --- a/cds/container/ts_deque_buffer.h +++ b/cds/container/ts_deque_buffer.h @@ -36,7 +36,7 @@ namespace cds { namespace container { std::atomic left; std::atomic right; std::atomic taken; - std::atomic data; + std::atomic data = {T()}; std::atomic timestamp[2]; // Insertion index, needed for the termination condition in // get_left_item. Items inserted at the left get negative @@ -56,7 +56,7 @@ namespace cds { namespace container { RandomEngine random_engine_; - std::atomic thread_id_counter_; + uint64_t global_thread_id_counter_; boost::thread_specific_ptr thread_id_; // Helper function to get index of thread @@ -65,17 +65,7 @@ namespace cds { namespace container { int* temp = thread_id_.get(); if (temp == NULL) { - int index = thread_id_counter_.load(); - if(index >= num_threads_) - { - return -1; - } - - while(!thread_id_counter_.compare_exchange_strong(index, index + 1)) - { - index = thread_id_counter_.load(); - } - + int index = __sync_fetch_and_add(&global_thread_id_counter_, 1); thread_id_.reset(new int (index)); return index; } @@ -182,7 +172,7 @@ namespace cds { namespace container { public: TSDequeBuffer(uint64_t num_threads, TimeStamp *timestamping) { - thread_id_counter_.store(0); + global_thread_id_counter_ = 0; num_threads_ = num_threads; timestamping_ = timestamping; @@ -205,7 +195,6 @@ namespace cds { namespace container { // Add a sentinal node. Item *new_item = new Item(); timestamping_->init_sentinel_atomic(new_item->timestamp); - new_item->data.store(0); new_item->taken.store(1); new_item->left.store(new_item); new_item->right.store(new_item); @@ -397,8 +386,10 @@ namespace cds { namespace container { } } - bool try_remove_left(T *element, uint64_t *invocation_time) + bool try_remove_left(T *element, uint64_t *invocation_time, bool *empty_check) { + *empty_check = false; + // Initialize the data needed for the emptiness check. uint64_t thread_id = get_thread_id(); Item **emptiness_check_left = @@ -539,12 +530,14 @@ namespace cds { namespace container { } } - element = NULL; + *empty_check = true; return !empty; } - bool try_remove_right(T *element, uint64_t *invocation_time) + bool try_remove_right(T *element, uint64_t *invocation_time, bool *empty_check) { + *empty_check = false; + // Initialize the data needed for the emptiness check. uint64_t thread_id = get_thread_id(); Item **emptiness_check_left = @@ -665,7 +658,7 @@ namespace cds { namespace container { } } - element = NULL; + *empty_check = true; return !empty; } }; From ff40ba77c0adc20a0a74acef4b6e23b4a345d86e Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Fri, 28 Dec 2018 03:18:12 +0300 Subject: [PATCH 26/29] stress tests added --- test/stress/queue/pop.cpp | 1 + test/stress/queue/push.cpp | 1 + test/stress/queue/push_pop.cpp | 1 + test/stress/queue/queue_type.h | 74 ++++++++++++++++++++++++++++++++++ test/stress/queue/random.cpp | 1 + test/stress/stack/push.cpp | 1 + test/stress/stack/push_pop.cpp | 1 + test/stress/stack/stack_type.h | 59 ++++++++++++++++++++++++++- test/unit/deque/ts_deque.cpp | 2 +- 9 files changed, 139 insertions(+), 2 deletions(-) diff --git a/test/stress/queue/pop.cpp b/test/stress/queue/pop.cpp index 5dbb72cd5..845c93b2d 100644 --- a/test/stress/queue/pop.cpp +++ b/test/stress/queue/pop.cpp @@ -149,6 +149,7 @@ namespace { CDSSTRESS_FCDeque( queue_pop ) CDSSTRESS_RWQueue( queue_pop ) CDSSTRESS_StdQueue( queue_pop ) + CDSSTRESS_TSDeque( queue_pop ) #undef CDSSTRESS_Queue_F #define CDSSTRESS_Queue_F( test_fixture, type_name ) \ diff --git a/test/stress/queue/push.cpp b/test/stress/queue/push.cpp index c2e80bba2..3b7b608e6 100644 --- a/test/stress/queue/push.cpp +++ b/test/stress/queue/push.cpp @@ -152,6 +152,7 @@ namespace { CDSSTRESS_FCDeque( queue_push ) CDSSTRESS_RWQueue( queue_push ) CDSSTRESS_StdQueue( queue_push ) + CDSSTRESS_TSDeque( queue_push ) #undef CDSSTRESS_Queue_F #define CDSSTRESS_Queue_F( test_fixture, type_name ) \ diff --git a/test/stress/queue/push_pop.cpp b/test/stress/queue/push_pop.cpp index 4803f5238..b855881d4 100644 --- a/test/stress/queue/push_pop.cpp +++ b/test/stress/queue/push_pop.cpp @@ -329,6 +329,7 @@ namespace { CDSSTRESS_FCDeque_HeavyValue( fc_with_heavy_value ) CDSSTRESS_RWQueue( simple_queue_push_pop ) CDSSTRESS_StdQueue( simple_queue_push_pop ) + CDSSTRESS_TSDeque( simple_queue_push_pop ) #undef CDSSTRESS_Queue_F #define CDSSTRESS_Queue_F( test_fixture, type_name ) \ diff --git a/test/stress/queue/queue_type.h b/test/stress/queue/queue_type.h index 4dd0f454b..3a80dc749 100644 --- a/test/stress/queue/queue_type.h +++ b/test/stress/queue/queue_type.h @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include @@ -102,6 +104,60 @@ namespace queue { } }; + template + class TSDequeL: public cds::container::TSDeque + { + typedef cds::container::TSDeque base_class; + public: + TSDequeL() : base_class( 17, 0 ) + {} + + bool push( T const& v ) + { + return base_class::insert_left( v ); + } + bool enqueue( T const& v ) + { + return push( v ); + } + + bool pop( T& v ) + { + return base_class::remove_right( &v ); + } + bool deque( T& v ) + { + return pop(v); + } + }; + + template + class TSDequeR: public cds::container::TSDeque + { + typedef cds::container::TSDeque base_class; + public: + TSDequeR() : base_class( 17, 0 ) + {} + + bool push( T const& v ) + { + return base_class::insert_right( v ); + } + bool enqueue( T const& v ) + { + return push( v ); + } + + bool pop( T& v ) + { + return base_class::remove_left( &v ); + } + bool deque( T& v ) + { + return pop(v); + } + }; + } // namespace details namespace fc_details{ @@ -503,6 +559,20 @@ namespace fc_details{ typedef details::FCDequeR< Value, fc_details::traits_FCDeque_elimination, boost::container::deque > FCDequeR_boost_elimination; typedef details::FCDequeR< Value, fc_details::traits_FCDeque_elimination_stat, boost::container::deque > FCDequeR_boost_elimination_stat; + // TSDeque + struct traits_TSDeque_item_counter: + public cds::container::tsdeque::make_traits< + cds::opt::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + + typedef details::TSDequeL< Value, cds::container::HardwareTimestamp, traits_TSDeque_item_counter > TSDequeL_hardware_timestamp; + typedef details::TSDequeL< Value, cds::container::HardwareIntervalTimestamp, traits_TSDeque_item_counter > TSDequeL_hardware_interval_timestamp; + typedef details::TSDequeL< Value, cds::container::AtomicCounterTimestamp, traits_TSDeque_item_counter > TSDequeL_atomic_counter_timestamp; + typedef details::TSDequeR< Value, cds::container::HardwareTimestamp, traits_TSDeque_item_counter > TSDequeR_hardware_timestamp; + typedef details::TSDequeR< Value, cds::container::HardwareIntervalTimestamp, traits_TSDeque_item_counter > TSDequeR_hardware_interval_timestamp; + typedef details::TSDequeR< Value, cds::container::AtomicCounterTimestamp, traits_TSDeque_item_counter > TSDequeR_atomic_counter_timestamp; + // STL typedef StdQueue_deque StdQueue_deque_Spinlock; typedef StdQueue_list StdQueue_list_Spinlock; @@ -831,6 +901,10 @@ namespace cds_test { CDSSTRESS_FCQueue_F( test_fixture, FCDequeL_HeavyValue_stat ) \ CDSSTRESS_FCDeque_HeavyValue_1( test_fixture ) +#define CDSSTRESS_TSDeque( test_fixture ) \ + CDSSTRESS_Queue_F( test_fixture, TSDequeL_hardware_timestamp ) \ + CDSSTRESS_Queue_F( test_fixture, TSDequeR_hardware_timestamp ) + #define CDSSTRESS_RWQueue( test_fixture ) \ CDSSTRESS_Queue_F( test_fixture, RWQueue_Spin ) \ CDSSTRESS_Queue_F( test_fixture, RWQueue_mutex ) \ diff --git a/test/stress/queue/random.cpp b/test/stress/queue/random.cpp index 357737d9e..029cb9422 100644 --- a/test/stress/queue/random.cpp +++ b/test/stress/queue/random.cpp @@ -216,6 +216,7 @@ namespace { CDSSTRESS_FCDeque( queue_random ) CDSSTRESS_RWQueue( queue_random ) CDSSTRESS_StdQueue( queue_random ) + CDSSTRESS_TSDeque( queue_random ) #undef CDSSTRESS_Queue_F #define CDSSTRESS_Queue_F( test_fixture, type_name ) \ diff --git a/test/stress/stack/push.cpp b/test/stress/stack/push.cpp index ae582aae7..f895570d2 100644 --- a/test/stress/stack/push.cpp +++ b/test/stress/stack/push.cpp @@ -180,5 +180,6 @@ namespace { CDSSTRESS_FCStack( stack_push ) CDSSTRESS_FCDeque( stack_push ) CDSSTRESS_StdStack( stack_push ) + CDSSTRESS_TSDeque( stack_push ) } // namespace diff --git a/test/stress/stack/push_pop.cpp b/test/stress/stack/push_pop.cpp index 74d8b8968..e47754576 100644 --- a/test/stress/stack/push_pop.cpp +++ b/test/stress/stack/push_pop.cpp @@ -257,5 +257,6 @@ namespace { CDSSTRESS_FCStack( stack_push_pop ) CDSSTRESS_FCDeque( stack_push_pop ) CDSSTRESS_StdStack( stack_push_pop ) + CDSSTRESS_TSDeque( stack_push_pop ) } // namespace diff --git a/test/stress/stack/stack_type.h b/test/stress/stack/stack_type.h index 4e58955ba..17c33fbe2 100644 --- a/test/stress/stack/stack_type.h +++ b/test/stress/stack/stack_type.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include @@ -78,6 +80,44 @@ namespace stack { } }; + template + class TSDequeL: public cds::container::TSDeque + { + typedef cds::container::TSDeque base_class; + public: + TSDequeL() : base_class( 9, 0 ) + {} + + bool push( T const& v ) + { + return base_class::insert_left( v ); + } + + bool pop( T& v ) + { + return base_class::remove_left( &v ); + } + }; + + template + class TSDequeR: public cds::container::TSDeque + { + typedef cds::container::TSDeque base_class; + public: + TSDequeR() : base_class( 9, 0 ) + {} + + bool push( T const& v ) + { + return base_class::insert_right( v ); + } + + bool pop( T& v ) + { + return base_class::remove_right( &v ); + } + }; + template < typename T, typename Stack, typename Lock> class StdStack { @@ -344,7 +384,7 @@ namespace stack { typedef cds::container::FCStack< T, std::stack >, traits_FCStack_elimination > FCStack_list_elimination; typedef cds::container::FCStack< T, std::stack >, traits_FCStack_elimination_stat > FCStack_list_elimination_stat; - // FCDeque + // FCDeque struct traits_FCDeque_stat: public cds::container::fcdeque::make_traits< cds::opt::stat< cds::container::fcdeque::stat<> > @@ -380,6 +420,19 @@ namespace stack { typedef details::FCDequeR< T, traits_FCDeque_elimination > FCDequeR_elimination; typedef details::FCDequeR< T, traits_FCDeque_elimination_stat > FCDequeR_elimination_stat; + // TSDeque + struct traits_TSDeque_item_counter: + public cds::container::tsdeque::make_traits< + cds::opt::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + + typedef details::TSDequeL< T, cds::container::HardwareTimestamp, traits_TSDeque_item_counter > TSDequeL_hardware_timestamp; + typedef details::TSDequeL< T, cds::container::HardwareIntervalTimestamp, traits_TSDeque_item_counter > TSDequeL_hardware_interval_timestamp; + typedef details::TSDequeL< T, cds::container::AtomicCounterTimestamp, traits_TSDeque_item_counter > TSDequeL_atomic_counter_timestamp; + typedef details::TSDequeR< T, cds::container::HardwareTimestamp, traits_TSDeque_item_counter > TSDequeR_hardware_timestamp; + typedef details::TSDequeR< T, cds::container::HardwareIntervalTimestamp, traits_TSDeque_item_counter > TSDequeR_hardware_interval_timestamp; + typedef details::TSDequeR< T, cds::container::AtomicCounterTimestamp, traits_TSDeque_item_counter > TSDequeR_atomic_counter_timestamp; // std::stack typedef details::StdStack< T, std::stack< T >, std::mutex > StdStack_Deque_Mutex; @@ -532,6 +585,10 @@ namespace cds_test { CDSSTRESS_Stack_F( test_fixture, FCDequeR_elimination ) \ CDSSTRESS_Stack_F( test_fixture, FCDequeR_elimination_stat ) +#define CDSSTRESS_TSDeque( test_fixture ) \ + CDSSTRESS_Stack_F( test_fixture, TSDequeL_hardware_timestamp ) \ + CDSSTRESS_Stack_F( test_fixture, TSDequeR_hardware_timestamp ) + #define CDSSTRESS_StdStack( test_fixture ) \ CDSSTRESS_Stack_F( test_fixture, StdStack_Deque_Mutex ) \ CDSSTRESS_Stack_F( test_fixture, StdStack_Deque_Spin ) \ diff --git a/test/unit/deque/ts_deque.cpp b/test/unit/deque/ts_deque.cpp index 808efa5c7..8be300b35 100644 --- a/test/unit/deque/ts_deque.cpp +++ b/test/unit/deque/ts_deque.cpp @@ -97,7 +97,7 @@ namespace { >::type > deque_type; - deque_type dq(1, 1000); + deque_type dq(1, 100000); test( dq ); } From 5ee65b581c03e4ac40187b103aa67385cb4959b2 Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Wed, 16 Jan 2019 23:03:11 +0300 Subject: [PATCH 27/29] fix tests for different architectures --- test/stress/queue/queue_type.h | 4 ++-- test/stress/stack/stack_type.h | 4 ++-- test/unit/deque/ts_deque.cpp | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/stress/queue/queue_type.h b/test/stress/queue/queue_type.h index 3a80dc749..79f5feb8c 100644 --- a/test/stress/queue/queue_type.h +++ b/test/stress/queue/queue_type.h @@ -902,8 +902,8 @@ namespace cds_test { CDSSTRESS_FCDeque_HeavyValue_1( test_fixture ) #define CDSSTRESS_TSDeque( test_fixture ) \ - CDSSTRESS_Queue_F( test_fixture, TSDequeL_hardware_timestamp ) \ - CDSSTRESS_Queue_F( test_fixture, TSDequeR_hardware_timestamp ) + CDSSTRESS_Queue_F( test_fixture, TSDequeL_atomic_counter_timestamp ) \ + CDSSTRESS_Queue_F( test_fixture, TSDequeR_atomic_counter_timestamp ) #define CDSSTRESS_RWQueue( test_fixture ) \ CDSSTRESS_Queue_F( test_fixture, RWQueue_Spin ) \ diff --git a/test/stress/stack/stack_type.h b/test/stress/stack/stack_type.h index 17c33fbe2..5490fd338 100644 --- a/test/stress/stack/stack_type.h +++ b/test/stress/stack/stack_type.h @@ -586,8 +586,8 @@ namespace cds_test { CDSSTRESS_Stack_F( test_fixture, FCDequeR_elimination_stat ) #define CDSSTRESS_TSDeque( test_fixture ) \ - CDSSTRESS_Stack_F( test_fixture, TSDequeL_hardware_timestamp ) \ - CDSSTRESS_Stack_F( test_fixture, TSDequeR_hardware_timestamp ) + CDSSTRESS_Stack_F( test_fixture, TSDequeL_atomic_counter_timestamp ) \ + CDSSTRESS_Stack_F( test_fixture, TSDequeR_atomic_counter_timestamp ) #define CDSSTRESS_StdStack( test_fixture ) \ CDSSTRESS_Stack_F( test_fixture, StdStack_Deque_Mutex ) \ diff --git a/test/unit/deque/ts_deque.cpp b/test/unit/deque/ts_deque.cpp index 8be300b35..61468f0d7 100644 --- a/test/unit/deque/ts_deque.cpp +++ b/test/unit/deque/ts_deque.cpp @@ -77,6 +77,8 @@ namespace { } }; + #if defined(CDS_ts_hardwaretimestamp_hwptime_defined) && defined(CDS_ts_hardwaretimestamp_hwtime_defined) + TEST_F( TSDeque, hardware_timestamping ) { typedef cds::container::TSDeque Date: Wed, 16 Jan 2019 23:46:35 +0300 Subject: [PATCH 28/29] fix rdtsc and rdtscp for i386 arch --- cds/compiler/gcc/amd64/ts_hardwaretimestamp.h | 8 ++++---- cds/compiler/gcc/x86/ts_hardwaretimestamp.h | 13 ++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h b/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h index 7cbc40709..acd1325e6 100644 --- a/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h +++ b/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h @@ -11,16 +11,16 @@ namespace cds { namespace tshardwaretimestamp { { uint64_t aux; uint64_t rax, rdx; - asm volatile ("rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); + __asm__ volatile ("rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); return (rdx << 32) + rax; } # define CDS_ts_hardwaretimestamp_hwtime_defined static inline uint64_t get_hwtime() { - unsigned int hi, lo; - __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); - return ((uint64_t)lo) | (((uint64_t)hi) << 32); + uint64_t high, low; + __asm__ volatile("rdtsc" : "=a"(low), "=d"(high)); + return ((uint64_t)low) | (((uint64_t)high) << 32); } }} // namespace gcc::amd64 diff --git a/cds/compiler/gcc/x86/ts_hardwaretimestamp.h b/cds/compiler/gcc/x86/ts_hardwaretimestamp.h index 1e14dfcc5..983fe2a2f 100644 --- a/cds/compiler/gcc/x86/ts_hardwaretimestamp.h +++ b/cds/compiler/gcc/x86/ts_hardwaretimestamp.h @@ -9,18 +9,17 @@ namespace cds { namespace tshardwaretimestamp { # define CDS_ts_hardwaretimestamp_hwptime_defined static inline uint64_t get_hwptime() { - uint64_t aux; - uint64_t rax, rdx; - asm volatile ("rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); - return (rdx << 32) + rax; + uint64_t ret; + __asm__ volatile ("rdtscp\n" : "=A" (ret) : : "ecx"); + return ret; } # define CDS_ts_hardwaretimestamp_hwtime_defined static inline uint64_t get_hwtime() { - unsigned int hi, lo; - __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); - return ((uint64_t)lo) | (((uint64_t)hi) << 32); + uint64_t ret; + __asm__ volatile("rdtsc" : "=A"(ret)); + return ret; } }} // namespace gcc::x86 From 2410e88660a85d6601c591f03df3006f6fc7971e Mon Sep 17 00:00:00 2001 From: Alexey Chirukhin Date: Fri, 18 Jan 2019 22:18:17 +0300 Subject: [PATCH 29/29] add rdtscp check --- cds/compiler/gcc/amd64/ts_hardwaretimestamp.h | 10 +++++ cds/compiler/gcc/x86/ts_hardwaretimestamp.h | 10 +++++ test/unit/deque/ts_deque.cpp | 39 +++++++++++-------- 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h b/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h index acd1325e6..278170d5a 100644 --- a/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h +++ b/cds/compiler/gcc/amd64/ts_hardwaretimestamp.h @@ -2,6 +2,7 @@ #define CDSLIB_COMPILER_GCC_AMD64_TS_HARDWARETIMESTAMP_H #include +#include namespace cds { namespace tshardwaretimestamp { namespace gcc { namespace amd64 { @@ -23,6 +24,15 @@ namespace cds { namespace tshardwaretimestamp { return ((uint64_t)low) | (((uint64_t)high) << 32); } + static inline int has_rdtscp() + { + unsigned int eax, ebx, ecx, edx; + if (__get_cpuid(0x80000001, &eax, &ebx, &ecx, &edx)) + return (edx >> 27) & 0x1; + else + return 0; + } + }} // namespace gcc::amd64 namespace platform { diff --git a/cds/compiler/gcc/x86/ts_hardwaretimestamp.h b/cds/compiler/gcc/x86/ts_hardwaretimestamp.h index 983fe2a2f..ee269339e 100644 --- a/cds/compiler/gcc/x86/ts_hardwaretimestamp.h +++ b/cds/compiler/gcc/x86/ts_hardwaretimestamp.h @@ -2,6 +2,7 @@ #define CDSLIB_COMPILER_GCC_X86_TS_HARDWARETIMESTAMP_H #include +#include namespace cds { namespace tshardwaretimestamp { namespace gcc { namespace x86 { @@ -22,6 +23,15 @@ namespace cds { namespace tshardwaretimestamp { return ret; } + static inline int has_rdtscp() + { + unsigned int eax, ebx, ecx, edx; + if (__get_cpuid(0x80000001, &eax, &ebx, &ecx, &edx)) + return (edx >> 27) & 0x1; + else + return 0; + } + }} // namespace gcc::x86 namespace platform { diff --git a/test/unit/deque/ts_deque.cpp b/test/unit/deque/ts_deque.cpp index 61468f0d7..0d35ce765 100644 --- a/test/unit/deque/ts_deque.cpp +++ b/test/unit/deque/ts_deque.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace { @@ -81,26 +82,32 @@ namespace { TEST_F( TSDeque, hardware_timestamping ) { - typedef cds::container::TSDeque - >::type - > deque_type; - - deque_type dq(1, 0); - test( dq ); + if (cds::tshardwaretimestamp::platform::has_rdtscp() == 1) + { + typedef cds::container::TSDeque + >::type + > deque_type; + + deque_type dq(1, 0); + test( dq ); + } } TEST_F( TSDeque, hardware_interval_timestamping ) { - typedef cds::container::TSDeque - >::type - > deque_type; - - deque_type dq(1, 100000); - test( dq ); + if (cds::tshardwaretimestamp::platform::has_rdtscp() == 1) + { + typedef cds::container::TSDeque + >::type + > deque_type; + + deque_type dq(1, 100000); + test( dq ); + } } #endif