From 475d5eb827c8a82b1450117cdcb903d62ba0b1f7 Mon Sep 17 00:00:00 2001
From: Cedric Roux <cedric.roux@eurecom.fr>
Date: Fri, 17 Aug 2018 12:16:58 +0200
Subject: [PATCH] T: bug fixes for the T cache

- 'busy' has to be handled in a more atomic fashion to avoid crazy
  runtime race conditions
- the basic simulator is too fast sometimes; rewrite the accesses to
  the T cache to avoid there again crazy behaviors

Hopefully that's better...
---
 common/utils/T/T.c            |  6 ++++++
 common/utils/T/T.h            | 39 ++++++++++++++++++++++++++++-------
 common/utils/T/T_defs.h       |  4 ++++
 common/utils/T/local_tracer.c |  2 +-
 4 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/common/utils/T/T.c b/common/utils/T/T.c
index 1cda8b7504..fbd3f75a0c 100644
--- a/common/utils/T/T.c
+++ b/common/utils/T/T.c
@@ -31,6 +31,12 @@ volatile int _T_freelist_head;
 volatile int *T_freelist_head = &_T_freelist_head;
 T_cache_t *T_cache;
 
+#if BASIC_SIMULATOR
+/* global variables used by T_GET_SLOT, see in T.h */
+volatile uint64_t T_next_id;
+volatile uint64_t T_active_id;
+#endif
+
 static void get_message(int s)
 {
   char t;
diff --git a/common/utils/T/T.h b/common/utils/T/T.h
index f1aa6ecd1d..18e4a590d9 100644
--- a/common/utils/T/T.h
+++ b/common/utils/T/T.h
@@ -112,13 +112,37 @@ extern volatile int *T_freelist_head;
 extern T_cache_t *T_cache;
 extern int *T_active;
 /* When running the basic simulator, we may fill the T cache too fast.
- * Let's not crash if it's full, just wait.
+ * Let's serialize write accesses to the T cache. For that, we use a
+ * 'ticket' mechanism. To acquire a T slot the caller needs to own the
+ * current active ticket. We also wait for the slot to be free if
+ * it is already in use.
  */
 #if BASIC_SIMULATOR
-#  define T_BASIC_SIMULATOR_WAIT \
-     while (T_cache[T_LOCAL_slot].busy) usleep(100)
+#  define T_GET_SLOT \
+     do { \
+       extern volatile uint64_t T_next_id; \
+       extern volatile uint64_t T_active_id; \
+       uint64_t id; \
+       /* get a ticket */ \
+       id = __sync_fetch_and_add(&T_next_id, 1); \
+       /* wait for our turn */ \
+       while (id != __sync_fetch_and_add(&T_active_id, 0)) /* busy wait */; \
+       /* this is our turn, try to acquire the slot until it's free */ \
+       do { \
+         T_LOCAL_busy = __sync_fetch_and_or(&T_cache[T_LOCAL_slot].busy, 0x01); \
+         if (T_LOCAL_busy & 0x01) usleep(100); \
+       } while (T_LOCAL_busy & 0x01); \
+       /* check that there are still some tickets */ \
+       if (__sync_fetch_and_add(&T_active_id, 0) == 0xffffffffffffffff) { \
+         printf("T: reached the end of times, bye...\n"); \
+         abort(); \
+       } \
+       /* free our ticket, which signals the next waiter that it's its turn */ \
+       (void)__sync_fetch_and_add(&T_active_id, 1); \
+     } while (0)
 #else
-#  define T_BASIC_SIMULATOR_WAIT /* */
+#  define T_GET_SLOT \
+     T_LOCAL_busy = __sync_fetch_and_or(&T_cache[T_LOCAL_slot].busy, 0x01);
 #endif
 
 /* used at header of Tn, allocates buffer */
@@ -126,11 +150,12 @@ extern int *T_active;
   char *T_LOCAL_buf; \
   int T_LOCAL_size = 0; \
   int T_LOCAL_slot; \
+  int T_LOCAL_busy; \
   T_LOCAL_slot = __sync_fetch_and_add(T_freelist_head, 1) \
                  & (T_CACHE_SIZE - 1); \
   (void)__sync_fetch_and_and(T_freelist_head, T_CACHE_SIZE - 1); \
-  T_BASIC_SIMULATOR_WAIT; \
-  if (T_cache[T_LOCAL_slot].busy) { \
+  T_GET_SLOT; \
+  if (T_LOCAL_busy & 0x01) { \
     printf("%s:%d:%s: T cache is full - consider increasing its size\n", \
            __FILE__, __LINE__, __FUNCTION__); \
     abort(); \
@@ -142,7 +167,7 @@ extern int *T_active;
 #define T_COMMIT() \
   T_cache[T_LOCAL_slot].length = T_LOCAL_size; \
   __sync_synchronize(); \
-  T_cache[T_LOCAL_slot].busy = 1; \
+  (void)__sync_fetch_and_or(&T_cache[T_LOCAL_slot].busy, 0x02);
 
 #define T_CHECK_SIZE(len, argnum) \
   if (T_LOCAL_size + (len) > T_BUFFER_MAX) { \
diff --git a/common/utils/T/T_defs.h b/common/utils/T/T_defs.h
index 6fe11bcfc3..83a380662d 100644
--- a/common/utils/T/T_defs.h
+++ b/common/utils/T/T_defs.h
@@ -31,6 +31,10 @@
 #endif
 
 typedef struct {
+  /* 'busy' is a bit field
+   * bit 0: 1 means that slot is acquired by writer
+   * bit 1: 1 means that slot is ready for consumption
+   */
   volatile int busy;
   char buffer[T_BUFFER_MAX];
   int length;
diff --git a/common/utils/T/local_tracer.c b/common/utils/T/local_tracer.c
index 19d0a4d717..f8e92881a9 100644
--- a/common/utils/T/local_tracer.c
+++ b/common/utils/T/local_tracer.c
@@ -337,7 +337,7 @@ static void forward(void *_forwarder, char *buf, int size)
 
 static void wait_message(void)
 {
-  while (T_local_cache[T_busylist_head].busy == 0) usleep(1000);
+  while ((T_local_cache[T_busylist_head].busy & 0x02) == 0) usleep(1000);
 }
 
 void T_local_tracer_main(int remote_port, int wait_for_tracer,