New threadpool
- removed some whitespace noise. - cleaned up CMakeLists.txt - reintegrated task_manager.c into thread-pool.c - cleaned up some unnecessary code Co-authored-by: Cedric Roux <cedric.roux@eurecom.fr> Co-authored-by: Mikel Irazabal <mikel.irazabal@openairinterface.com>
This commit is contained in:
Bartosz Podrygajlo
co-authored by
Cedric Roux
Mikel Irazabal
Cedric Roux
Mikel Irazabal
parent
c942fe43a9
commit
21dafbcde7
@@ -1,6 +1,6 @@
|
||||
if (ENABLE_TESTS)
|
||||
add_subdirectory(test)
|
||||
endif()
|
||||
add_library(thread-pool thread-pool.c)
|
||||
add_library(thread-pool thread-pool.c task_ans.c)
|
||||
target_link_libraries(thread-pool PRIVATE utils)
|
||||
target_include_directories(thread-pool PUBLIC .)
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
#ifndef BOUNDED_NOTIFIED_FIFO_H
|
||||
#define BOUNDED_NOTIFIED_FIFO_H
|
||||
|
||||
#include "assertions.h"
|
||||
#include <stdint.h>
|
||||
#include <memory.h>
|
||||
#include "task.h"
|
||||
#include <pthread.h>
|
||||
#include "pthread_utils.h"
|
||||
|
||||
// For working correctly, maintain the default elements to a 2^N e.g., 2^5=32
|
||||
#define DEFAULT_ELM 256
|
||||
|
||||
typedef struct seq_ring_buf_s {
|
||||
task_t* array;
|
||||
size_t cap;
|
||||
uint32_t head;
|
||||
uint32_t tail;
|
||||
_Atomic uint64_t sz;
|
||||
} seq_ring_task_t;
|
||||
|
||||
|
||||
static size_t size_seq_ring_task(seq_ring_task_t* r)
|
||||
{
|
||||
DevAssert(r != NULL);
|
||||
|
||||
return r->head - r->tail;
|
||||
}
|
||||
|
||||
static uint32_t mask(uint32_t cap, uint32_t val)
|
||||
{
|
||||
return val & (cap - 1);
|
||||
}
|
||||
|
||||
static bool full(seq_ring_task_t* r)
|
||||
{
|
||||
return size_seq_ring_task(r) == r->cap - 1;
|
||||
}
|
||||
|
||||
static void enlarge_buffer(seq_ring_task_t* r)
|
||||
{
|
||||
DevAssert(r != NULL);
|
||||
DevAssert(full(r));
|
||||
|
||||
const uint32_t factor = 2;
|
||||
task_t* tmp_buffer = calloc(r->cap * factor, sizeof(task_t));
|
||||
DevAssert(tmp_buffer != NULL);
|
||||
|
||||
const uint32_t head_pos = mask(r->cap, r->head);
|
||||
const uint32_t tail_pos = mask(r->cap, r->tail);
|
||||
|
||||
if (head_pos > tail_pos) {
|
||||
memcpy(tmp_buffer, r->array + tail_pos, (head_pos - tail_pos) * sizeof(task_t));
|
||||
} else {
|
||||
memcpy(tmp_buffer, r->array + tail_pos, (r->cap - tail_pos) * sizeof(task_t));
|
||||
memcpy(tmp_buffer + (r->cap - tail_pos), r->array, head_pos * sizeof(task_t));
|
||||
}
|
||||
r->cap *= factor;
|
||||
free(r->array);
|
||||
r->array = tmp_buffer;
|
||||
r->tail = 0;
|
||||
r->head = r->cap / 2 - 1;
|
||||
}
|
||||
|
||||
static void init_seq_ring_task(seq_ring_task_t* r)
|
||||
{
|
||||
DevAssert(r != NULL);
|
||||
task_t* tmp_buffer = calloc(DEFAULT_ELM, sizeof(task_t));
|
||||
DevAssert(tmp_buffer != NULL);
|
||||
seq_ring_task_t tmp = {.array = tmp_buffer, .head = 0, .tail = 0, .cap = DEFAULT_ELM};
|
||||
memcpy(r, &tmp, sizeof(seq_ring_task_t));
|
||||
r->sz = 0;
|
||||
}
|
||||
|
||||
static void free_seq_ring_task(seq_ring_task_t* r)
|
||||
{
|
||||
DevAssert(r != NULL);
|
||||
free(r->array);
|
||||
}
|
||||
|
||||
static void push_back_seq_ring_task(seq_ring_task_t* r, task_t t)
|
||||
{
|
||||
DevAssert(r != NULL);
|
||||
|
||||
if (full(r))
|
||||
enlarge_buffer(r);
|
||||
|
||||
const uint32_t pos = mask(r->cap, r->head);
|
||||
r->array[pos] = t;
|
||||
r->head += 1;
|
||||
r->sz += 1;
|
||||
}
|
||||
|
||||
static task_t pop_seq_ring_task(seq_ring_task_t* r)
|
||||
{
|
||||
DevAssert(r != NULL);
|
||||
DevAssert(size_seq_ring_task(r) > 0);
|
||||
|
||||
const uint32_t pos = mask(r->cap, r->tail);
|
||||
task_t t = r->array[pos];
|
||||
r->tail += 1;
|
||||
r->sz -= 1;
|
||||
return t;
|
||||
}
|
||||
|
||||
#undef DEFAULT_ELM
|
||||
|
||||
typedef struct {
|
||||
pthread_mutex_t mtx;
|
||||
pthread_cond_t cv;
|
||||
seq_ring_task_t r;
|
||||
size_t idx;
|
||||
} not_q_t;
|
||||
|
||||
typedef struct {
|
||||
task_t t;
|
||||
bool success;
|
||||
} ret_try_t;
|
||||
|
||||
static void init_not_q(not_q_t* q, size_t idx)
|
||||
{
|
||||
DevAssert(q != NULL);
|
||||
|
||||
q->idx = idx;
|
||||
|
||||
init_seq_ring_task(&q->r);
|
||||
|
||||
mutexinit(q->mtx);
|
||||
condinit(q->cv);
|
||||
}
|
||||
|
||||
static void free_not_q(not_q_t* q)
|
||||
{
|
||||
DevAssert(q != NULL);
|
||||
|
||||
free_seq_ring_task(&q->r);
|
||||
|
||||
mutexdestroy(q->mtx);
|
||||
conddestroy(q->cv);
|
||||
}
|
||||
|
||||
static bool try_push_not_q(not_q_t* q, task_t t)
|
||||
{
|
||||
DevAssert(q != NULL);
|
||||
|
||||
|
||||
if (mutextrylock(q->mtx) != 0)
|
||||
return false;
|
||||
|
||||
push_back_seq_ring_task(&q->r, t);
|
||||
|
||||
const size_t sz = size_seq_ring_task(&q->r);
|
||||
DevAssert(sz > 0);
|
||||
|
||||
mutexunlock(q->mtx);
|
||||
|
||||
condsignal(q->cv);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void push_not_q(not_q_t* q, task_t t)
|
||||
{
|
||||
DevAssert(q != NULL);
|
||||
DevAssert(t.func != NULL);
|
||||
|
||||
mutexlock(q->mtx);
|
||||
|
||||
push_back_seq_ring_task(&q->r, t);
|
||||
|
||||
DevAssert(size_seq_ring_task(&q->r) > 0);
|
||||
|
||||
mutexunlock(q->mtx);
|
||||
|
||||
condsignal(q->cv);
|
||||
}
|
||||
|
||||
static ret_try_t try_pop_not_q(not_q_t* q)
|
||||
{
|
||||
DevAssert(q != NULL);
|
||||
|
||||
ret_try_t ret = {.success = false};
|
||||
|
||||
int rc = mutextrylock(q->mtx);
|
||||
DevAssert(rc == 0 || rc == EBUSY);
|
||||
|
||||
if (rc == EBUSY)
|
||||
return ret;
|
||||
|
||||
size_t sz = size_seq_ring_task(&q->r);
|
||||
if (sz == 0) {
|
||||
mutexunlock(q->mtx);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
DevAssert(sz > 0);
|
||||
ret.t = pop_seq_ring_task(&q->r);
|
||||
|
||||
mutexunlock(q->mtx);
|
||||
ret.success = true;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static bool pop_not_q(not_q_t* q, ret_try_t* out)
|
||||
{
|
||||
DevAssert(q != NULL);
|
||||
DevAssert(out != NULL);
|
||||
|
||||
mutexlock(q->mtx);
|
||||
|
||||
while (size_seq_ring_task(&q->r) == 0) {
|
||||
condwait(q->cv, q->mtx);
|
||||
}
|
||||
|
||||
out->t = pop_seq_ring_task(&q->r);
|
||||
|
||||
mutexunlock(q->mtx);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,223 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
#ifndef NOTIFIED_FIFO_H
|
||||
#define NOTIFIED_FIFO_H
|
||||
#include "pthread_utils.h"
|
||||
#include <stdint.h>
|
||||
#include <pthread.h>
|
||||
#include "time_meas.h"
|
||||
#include <memory.h>
|
||||
#include <stdalign.h>
|
||||
#include "assertions.h"
|
||||
|
||||
/// @brief Element on the notifiedFifo_t
|
||||
/// next: internal FIFO chain, do not set it
|
||||
/// key: a long int that the client can use to identify a job or a group of messages
|
||||
/// ResponseFifo: if the client defines a response FIFO, the job will be posted back after processing
|
||||
/// processingFunc: any function (of type void processingFunc(void *)) that a worker will process
|
||||
/// msgData: the data passed to `processingFunc`. It can be added automatically, or you can set it to a buffer you are managing
|
||||
/// malloced: a boolean that enables internal free in the case of no return FIFO or abort feature
|
||||
typedef struct notifiedFIFO_elt_s {
|
||||
struct notifiedFIFO_elt_s *next;
|
||||
uint64_t key; // To filter out elements
|
||||
struct notifiedFIFO_s *reponseFifo;
|
||||
void (*processingFunc)(void *);
|
||||
bool malloced;
|
||||
oai_cputime_t creationTime;
|
||||
oai_cputime_t startProcessingTime;
|
||||
oai_cputime_t endProcessingTime;
|
||||
oai_cputime_t returnTime;
|
||||
// use alignas(32) to align msgData to 32b
|
||||
// user data behind it will be aligned to 32b as well
|
||||
// important! this needs to be the last member in the struct
|
||||
alignas(32) void *msgData;
|
||||
} notifiedFIFO_elt_t;
|
||||
|
||||
typedef struct notifiedFIFO_s {
|
||||
notifiedFIFO_elt_t *outF;
|
||||
notifiedFIFO_elt_t *inF;
|
||||
pthread_mutex_t lockF;
|
||||
pthread_cond_t notifF;
|
||||
bool abortFIFO; // if set, the FIFO always returns NULL -> abort condition
|
||||
} notifiedFIFO_t;
|
||||
|
||||
/// @brief Creates a new job.
|
||||
/// @param size The data part of the job will have this size
|
||||
/// @param key the job can be identified
|
||||
/// @param reponseFifo response fifo
|
||||
/// @param processingFunc function to call
|
||||
/// @return new notifiedFIFO_elt_t element with extra memory allocated at the end equal to size.
|
||||
static inline notifiedFIFO_elt_t *newNotifiedFIFO_elt(int size,
|
||||
uint64_t key,
|
||||
notifiedFIFO_t *reponseFifo,
|
||||
void (*processingFunc)(void *))
|
||||
{
|
||||
notifiedFIFO_elt_t *ret = (notifiedFIFO_elt_t *)memalign(32, sizeof(notifiedFIFO_elt_t) + size);
|
||||
AssertFatal(NULL != ret, "out of memory\n");
|
||||
ret->next = NULL;
|
||||
ret->key = key;
|
||||
ret->reponseFifo = reponseFifo;
|
||||
ret->processingFunc = processingFunc;
|
||||
// We set user data piece aligend 32 bytes to be able to process it with SIMD
|
||||
// msgData is aligned to 32bytes, so everything after will be as well
|
||||
ret->msgData = ((uint8_t *)ret) + sizeof(notifiedFIFO_elt_t);
|
||||
ret->malloced = true;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief Get pointer to the data carried by notifiedFIFO_elt_t
|
||||
/// @param elt
|
||||
/// @return void pointer to the data allocated with the message
|
||||
static inline void *NotifiedFifoData(notifiedFIFO_elt_t *elt)
|
||||
{
|
||||
return elt->msgData;
|
||||
}
|
||||
|
||||
/// @brief Delete a notifiedFIFO_elt_t if its allocated
|
||||
/// @param elt
|
||||
static inline void delNotifiedFIFO_elt(notifiedFIFO_elt_t *elt)
|
||||
{
|
||||
if (elt->malloced) {
|
||||
elt->malloced = false;
|
||||
free(elt);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void initNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf)
|
||||
{
|
||||
nf->inF = NULL;
|
||||
nf->outF = NULL;
|
||||
nf->abortFIFO = false;
|
||||
}
|
||||
static inline void initNotifiedFIFO(notifiedFIFO_t *nf)
|
||||
{
|
||||
mutexinit(nf->lockF);
|
||||
condinit(nf->notifF);
|
||||
initNotifiedFIFO_nothreadSafe(nf);
|
||||
// No delete function: the creator has only to free the memory
|
||||
}
|
||||
|
||||
static inline void pushNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg)
|
||||
{
|
||||
msg->next = NULL;
|
||||
|
||||
if (nf->outF == NULL)
|
||||
nf->outF = msg;
|
||||
|
||||
if (nf->inF != NULL)
|
||||
nf->inF->next = msg;
|
||||
|
||||
nf->inF = msg;
|
||||
}
|
||||
|
||||
static inline void pushNotifiedFIFO(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg)
|
||||
{
|
||||
mutexlock(nf->lockF);
|
||||
if (!nf->abortFIFO) {
|
||||
pushNotifiedFIFO_nothreadSafe(nf, msg);
|
||||
condsignal(nf->notifF);
|
||||
}
|
||||
mutexunlock(nf->lockF);
|
||||
}
|
||||
|
||||
static inline notifiedFIFO_elt_t *pullNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf)
|
||||
{
|
||||
if (nf->outF == NULL)
|
||||
return NULL;
|
||||
if (nf->abortFIFO)
|
||||
return NULL;
|
||||
|
||||
notifiedFIFO_elt_t *ret = nf->outF;
|
||||
|
||||
AssertFatal(nf->outF != nf->outF->next, "Circular list in thread pool: push several times the same buffer is forbidden\n");
|
||||
|
||||
nf->outF = nf->outF->next;
|
||||
|
||||
if (nf->outF == NULL)
|
||||
nf->inF = NULL;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline notifiedFIFO_elt_t *pullNotifiedFIFO(notifiedFIFO_t *nf)
|
||||
{
|
||||
mutexlock(nf->lockF);
|
||||
notifiedFIFO_elt_t *ret = NULL;
|
||||
|
||||
while ((ret = pullNotifiedFIFO_nothreadSafe(nf)) == NULL && !nf->abortFIFO)
|
||||
condwait(nf->notifF, nf->lockF);
|
||||
|
||||
mutexunlock(nf->lockF);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline notifiedFIFO_elt_t *pollNotifiedFIFO(notifiedFIFO_t *nf)
|
||||
{
|
||||
int tmp = mutextrylock(nf->lockF);
|
||||
|
||||
if (tmp != 0)
|
||||
return NULL;
|
||||
|
||||
if (nf->abortFIFO) {
|
||||
mutexunlock(nf->lockF);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
notifiedFIFO_elt_t *ret = pullNotifiedFIFO_nothreadSafe(nf);
|
||||
mutexunlock(nf->lockF);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline time_stats_t exec_time_stats_NotifiedFIFO(const notifiedFIFO_elt_t *elt)
|
||||
{
|
||||
time_stats_t ts = {0};
|
||||
if (elt->startProcessingTime == 0 && elt->endProcessingTime == 0)
|
||||
return ts; /* no measurements done */
|
||||
ts.in = elt->startProcessingTime;
|
||||
ts.diff = elt->endProcessingTime - ts.in;
|
||||
ts.p_time = ts.diff;
|
||||
ts.diff_square = ts.diff * ts.diff;
|
||||
ts.max = ts.diff;
|
||||
ts.trials = 1;
|
||||
return ts;
|
||||
}
|
||||
|
||||
// This functions aborts all messages in the queue, and marks the queue as
|
||||
// "aborted", such that every call to it will return NULL
|
||||
static inline void abortNotifiedFIFO(notifiedFIFO_t *nf)
|
||||
{
|
||||
mutexlock(nf->lockF);
|
||||
nf->abortFIFO = true;
|
||||
notifiedFIFO_elt_t **elt = &nf->outF;
|
||||
while (*elt != NULL) {
|
||||
notifiedFIFO_elt_t *p = *elt;
|
||||
*elt = (*elt)->next;
|
||||
delNotifiedFIFO_elt(p);
|
||||
}
|
||||
|
||||
if (nf->outF == NULL)
|
||||
nf->inF = NULL;
|
||||
condbroadcast(nf->notifF);
|
||||
mutexunlock(nf->lockF);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
#ifndef PTHREAD_UTILS_H
|
||||
#define PTHREAD_UTILS_H
|
||||
|
||||
#include <pthread.h>
|
||||
#define mutexinit(mutex) {int ret=pthread_mutex_init(&mutex,NULL); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define condinit(signal) {int ret=pthread_cond_init(&signal,NULL); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define mutexlock(mutex) {int ret=pthread_mutex_lock(&mutex); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define mutextrylock(mutex) pthread_mutex_trylock(&mutex)
|
||||
#define mutexunlock(mutex) {int ret=pthread_mutex_unlock(&mutex); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define condwait(condition, mutex) {int ret=pthread_cond_wait(&condition, &mutex); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define condbroadcast(signal) {int ret=pthread_cond_broadcast(&signal); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define condsignal(signal) {int ret=pthread_cond_signal(&signal); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define mutexdestroy(mutex) { int ret = pthread_mutex_destroy(&mutex);\
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define conddestroy(condition) { int ret = pthread_cond_destroy(&condition);\
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#endif
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
#ifndef TASK_WORK_STEALING_THREAD_POOL_H
|
||||
#define TASK_WORK_STEALING_THREAD_POOL_H
|
||||
|
||||
typedef struct {
|
||||
void* args;
|
||||
void (*func)(void* args);
|
||||
} task_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
#include "task_ans.h"
|
||||
#include "assertions.h"
|
||||
#include <limits.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
|
||||
void completed_task_ans(task_ans_t* task)
|
||||
{
|
||||
DevAssert(task != NULL);
|
||||
|
||||
if (atomic_load_explicit(&task->status, memory_order_acquire) != 0)
|
||||
AssertFatal(0, "Task already finished?");
|
||||
|
||||
atomic_store_explicit(&task->status, 1, memory_order_release);
|
||||
}
|
||||
|
||||
void join_task_ans(task_ans_t* arr, size_t len)
|
||||
{
|
||||
DevAssert(len < INT_MAX);
|
||||
DevAssert(arr != NULL);
|
||||
|
||||
// Spin lock inspired by:
|
||||
// The Art of Writing Efficient Programs:
|
||||
// An advanced programmer's guide to efficient hardware utilization
|
||||
// and compiler optimizations using C++ examples
|
||||
const struct timespec ns = {0, 1};
|
||||
uint64_t i = 0;
|
||||
int j = len - 1;
|
||||
for (; j != -1; i++) {
|
||||
for (; j != -1; --j) {
|
||||
int const task_completed = 1;
|
||||
if (atomic_load_explicit(&arr[j].status, memory_order_acquire) != task_completed)
|
||||
break;
|
||||
}
|
||||
if (i % 8 == 0) {
|
||||
nanosleep(&ns, NULL);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
#ifndef TASK_ANSWER_THREAD_POOL_H
|
||||
#define TASK_ANSWER_THREAD_POOL_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef __cplusplus
|
||||
#include <stdalign.h>
|
||||
#include <stdatomic.h>
|
||||
#else
|
||||
#include <atomic>
|
||||
#define _Atomic(X) std::atomic<X>
|
||||
#define _Alignas(X) alignas(X)
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#if defined(__i386__) || defined(__x86_64__)
|
||||
#define LEVEL1_DCACHE_LINESIZE 64
|
||||
#elif defined(__aarch64__)
|
||||
// This is not always true for ARM
|
||||
// in linux, you can obtain the size at runtime using sysconf (_SC_LEVEL1_DCACHE_LINESIZE)
|
||||
// or from the bash with the command $ getconf LEVEL1_DCACHE_LINESIZE
|
||||
// in c++ using std::hardware_destructive_interference_size
|
||||
#define LEVEL1_DCACHE_LINESIZE 64
|
||||
#else
|
||||
#error Unknown CPU architecture
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
// Avoid false sharing
|
||||
_Alignas(LEVEL1_DCACHE_LINESIZE) _Atomic(int) status;
|
||||
} task_ans_t;
|
||||
|
||||
typedef struct {
|
||||
uint8_t* buf;
|
||||
size_t len;
|
||||
size_t cap; // capacity
|
||||
task_ans_t* ans;
|
||||
} thread_info_tm_t;
|
||||
|
||||
|
||||
void join_task_ans(task_ans_t* arr, size_t len);
|
||||
|
||||
void completed_task_ans(task_ans_t* task);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -27,8 +27,10 @@
|
||||
#include <unistd.h>
|
||||
#include <ctype.h>
|
||||
#include <sys/sysinfo.h>
|
||||
#include <threadPool/thread-pool.h>
|
||||
#include "thread-pool.h"
|
||||
#include "log.h"
|
||||
#include "task_ans.h"
|
||||
#include "notified_fifo.h"
|
||||
|
||||
void displayList(notifiedFIFO_t *nf)
|
||||
{
|
||||
@@ -47,6 +49,7 @@ struct testData {
|
||||
int id;
|
||||
int sleepTime;
|
||||
char txt[50];
|
||||
task_ans_t* task_ans;
|
||||
};
|
||||
|
||||
void processing(void *arg)
|
||||
@@ -57,6 +60,7 @@ void processing(void *arg)
|
||||
in->sleepTime = rand() % 1000;
|
||||
usleep(in->sleepTime);
|
||||
// printf("done: %d, %s, in thr %ld\n",in->id, in->txt,pthread_self() );
|
||||
completed_task_ans(in->task_ans);
|
||||
}
|
||||
|
||||
int main()
|
||||
@@ -102,8 +106,6 @@ int main()
|
||||
tpool_t pool;
|
||||
char params[] = "1,2,3,4,5";
|
||||
initTpool(params, &pool, true);
|
||||
notifiedFIFO_t worker_back;
|
||||
initNotifiedFIFO(&worker_back);
|
||||
|
||||
//sleep(1);
|
||||
int cumulProcessTime = 0;
|
||||
@@ -112,21 +114,22 @@ int main()
|
||||
int nb_jobs = 4;
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
int parall = nb_jobs;
|
||||
task_ans_t task_ans[parall];
|
||||
memset(task_ans, 0, sizeof(task_ans));
|
||||
struct testData test_data[parall];
|
||||
memset(test_data, 0, sizeof(test_data));
|
||||
for (int j = 0; j < parall; j++) {
|
||||
notifiedFIFO_elt_t *work = newNotifiedFIFO_elt(sizeof(struct testData), i, &worker_back, processing);
|
||||
struct testData *x = (struct testData *)NotifiedFifoData(work);
|
||||
task_t task = {.args = &test_data[j], .func = processing};
|
||||
struct testData *x = (struct testData *)task.args;
|
||||
x->id = i;
|
||||
pushTpool(&pool, work);
|
||||
x->task_ans = &task_ans[j];
|
||||
pushTpool(&pool, task);
|
||||
}
|
||||
join_task_ans(task_ans, parall);
|
||||
int sleepmax = 0;
|
||||
while (parall) {
|
||||
tmp = pullTpool(&worker_back, &pool);
|
||||
if (tmp) {
|
||||
parall--;
|
||||
struct testData *dd = NotifiedFifoData(tmp);
|
||||
if (dd->sleepTime > sleepmax)
|
||||
sleepmax = dd->sleepTime;
|
||||
delNotifiedFIFO_elt(tmp);
|
||||
for (int j = 0; j < parall; j++) {
|
||||
if (test_data[j].sleepTime > sleepmax) {
|
||||
sleepmax = test_data[j].sleepTime;
|
||||
}
|
||||
}
|
||||
cumulProcessTime += sleepmax;
|
||||
|
||||
@@ -1,163 +1,192 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Author and copyright: Laurent Thomas, open-cells.com
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
#define _GNU_SOURCE
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <string.h>
|
||||
#include <ctype.h>
|
||||
#include <threadPool/thread-pool.h>
|
||||
#include "thread-pool.h"
|
||||
#include "bounded_notified_fifo.h"
|
||||
#include <sys/sysinfo.h>
|
||||
|
||||
typedef struct {
|
||||
tpool_t* tpool;
|
||||
int idx;
|
||||
} task_thread_args_t;
|
||||
|
||||
static inline notifiedFIFO_elt_t *pullNotifiedFifoRemember( notifiedFIFO_t *nf, struct one_thread *thr) {
|
||||
mutexlock(nf->lockF);
|
||||
|
||||
while(!nf->outF && !thr->terminate)
|
||||
condwait(nf->notifF, nf->lockF);
|
||||
|
||||
if (thr->terminate) {
|
||||
mutexunlock(nf->lockF);
|
||||
return NULL;
|
||||
void pushTpool(tpool_t* tpool, task_t task)
|
||||
{
|
||||
DevAssert(tpool != NULL);
|
||||
if (tpool->len_thr == 0) {
|
||||
task.func(task.args);
|
||||
return;
|
||||
}
|
||||
|
||||
notifiedFIFO_elt_t *ret=nf->outF;
|
||||
nf->outF=nf->outF->next;
|
||||
size_t const index = tpool->index++;
|
||||
size_t const len_thr = tpool->len_thr;
|
||||
|
||||
if (nf->outF==NULL)
|
||||
nf->inF=NULL;
|
||||
not_q_t* q_arr = (not_q_t*)tpool->q_arr;
|
||||
|
||||
// For abort feature
|
||||
thr->runningOnKey=ret->key;
|
||||
thr->dropJob = false;
|
||||
mutexunlock(nf->lockF);
|
||||
return ret;
|
||||
for (size_t i = 0; i < len_thr; ++i) {
|
||||
if (try_push_not_q(&q_arr[(i + index) % len_thr], task)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
push_not_q(&q_arr[index % len_thr], task);
|
||||
}
|
||||
|
||||
void *one_thread(void *arg) {
|
||||
struct one_thread *myThread=(struct one_thread *) arg;
|
||||
struct thread_pool *tp=myThread->pool;
|
||||
static void* worker_thread(void* arg)
|
||||
{
|
||||
DevAssert(arg != NULL);
|
||||
|
||||
// Infinite loop to process requests
|
||||
do {
|
||||
notifiedFIFO_elt_t *elt=pullNotifiedFifoRemember(&tp->incomingFifo, myThread);
|
||||
if (elt == NULL) {
|
||||
AssertFatal(myThread->terminate, "pullNotifiedFifoRemember() returned NULL although thread not aborted\n");
|
||||
task_thread_args_t* args = (task_thread_args_t*)arg;
|
||||
int const idx = args->idx;
|
||||
tpool_t* tpool = args->tpool;
|
||||
|
||||
uint32_t const len = tpool->len_thr;
|
||||
uint32_t const num_it = 2 * (tpool->len_thr + idx);
|
||||
|
||||
not_q_t* q_arr = (not_q_t*)tpool->q_arr;
|
||||
|
||||
init_not_q(&q_arr[idx], idx);
|
||||
// Synchronize all threads
|
||||
pthread_barrier_wait(&tpool->barrier);
|
||||
|
||||
for (;;) {
|
||||
ret_try_t ret = {.success = false};
|
||||
|
||||
for (uint32_t i = idx; i < num_it; ++i) {
|
||||
ret = try_pop_not_q(&q_arr[i % len]);
|
||||
if (ret.success == true)
|
||||
break;
|
||||
}
|
||||
|
||||
if (ret.success == false) {
|
||||
if (pop_not_q(&q_arr[idx], &ret) == false)
|
||||
break;
|
||||
}
|
||||
|
||||
if (ret.t.func == NULL && ret.t.args == NULL) {
|
||||
pushTpool(tpool, (task_t){.args = NULL, .func = NULL});
|
||||
break;
|
||||
}
|
||||
ret.t.func(ret.t.args);
|
||||
}
|
||||
|
||||
if (tp->measurePerf) elt->startProcessingTime=rdtsc_oai();
|
||||
|
||||
elt->processingFunc(NotifiedFifoData(elt));
|
||||
|
||||
if (tp->measurePerf) elt->endProcessingTime=rdtsc_oai();
|
||||
|
||||
if (elt->reponseFifo) {
|
||||
// Check if the job is still alive, else it has been aborted
|
||||
mutexlock(tp->incomingFifo.lockF);
|
||||
|
||||
if (myThread->dropJob)
|
||||
delNotifiedFIFO_elt(elt);
|
||||
else
|
||||
pushNotifiedFIFO(elt->reponseFifo, elt);
|
||||
myThread->runningOnKey=-1;
|
||||
mutexunlock(tp->incomingFifo.lockF);
|
||||
}
|
||||
else
|
||||
delNotifiedFIFO_elt(elt);
|
||||
} while (!myThread->terminate);
|
||||
free(args);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void initNamedTpool(char *params,tpool_t *pool, bool performanceMeas, char *name) {
|
||||
memset(pool,0,sizeof(*pool));
|
||||
char *measr=getenv("OAI_THREADPOOLMEASUREMENTS");
|
||||
pool->measurePerf=performanceMeas;
|
||||
// force measurement if the output is defined
|
||||
pool->measurePerf |= measr!=NULL;
|
||||
void initNamedTpool(char* params, tpool_t* tpool, bool performanceMeas, char* name)
|
||||
{
|
||||
(void)performanceMeas;
|
||||
|
||||
if (measr) {
|
||||
mkfifo(measr,0666);
|
||||
AssertFatal(-1 != (pool->dummyKeepReadingTraceFd=
|
||||
open(measr, O_RDONLY| O_NONBLOCK)),"");
|
||||
AssertFatal(-1 != (pool->traceFd=
|
||||
open(measr, O_WRONLY|O_APPEND|O_NOATIME|O_NONBLOCK)),"");
|
||||
} else
|
||||
pool->traceFd=-1;
|
||||
DevAssert(tpool != NULL);
|
||||
memset(tpool, 0, sizeof(*tpool));
|
||||
|
||||
pool->activated=true;
|
||||
initNotifiedFIFO(&pool->incomingFifo);
|
||||
char *saveptr, * curptr;
|
||||
char *parms_cpy=strdup(params);
|
||||
pool->nbThreads=0;
|
||||
curptr=strtok_r(parms_cpy,",",&saveptr);
|
||||
struct one_thread * ptr;
|
||||
char *tname = (name == NULL ? "Tpool" : name);
|
||||
while ( curptr!=NULL ) {
|
||||
int c=toupper(curptr[0]);
|
||||
char* tname = (name == NULL ? "Tpool" : name);
|
||||
char *saveptr, *curptr;
|
||||
char* parms_cpy = strdup(params);
|
||||
curptr = strtok_r(parms_cpy, ",", &saveptr);
|
||||
int core_id[128] = {0};
|
||||
int num_workers = 0;
|
||||
while (curptr != NULL) {
|
||||
int c = toupper(curptr[0]);
|
||||
|
||||
switch (c) {
|
||||
|
||||
case 'N':
|
||||
pool->activated=false;
|
||||
break;
|
||||
|
||||
default:
|
||||
ptr=pool->allthreads;
|
||||
pool->allthreads=(struct one_thread *)malloc(sizeof(struct one_thread));
|
||||
pool->allthreads->next=ptr;
|
||||
pool->allthreads->coreID=atoi(curptr);
|
||||
pool->allthreads->id=pool->nbThreads;
|
||||
pool->allthreads->pool=pool;
|
||||
pool->allthreads->dropJob = false;
|
||||
pool->allthreads->terminate = false;
|
||||
//Configure the thread scheduler policy for Linux
|
||||
// set the thread name for debugging
|
||||
sprintf(pool->allthreads->name,"%s%d_%d",tname,pool->nbThreads,pool->allthreads->coreID);
|
||||
// we set the maximum priority for thread pool threads (which is close
|
||||
// but not equal to Linux maximum). See also the corresponding commit
|
||||
// message; initially introduced for O-RAN 7.2 fronthaul split
|
||||
threadCreate(&pool->allthreads->threadID, one_thread, (void *)pool->allthreads,
|
||||
pool->allthreads->name, pool->allthreads->coreID, OAI_PRIORITY_RT_MAX);
|
||||
pool->nbThreads++;
|
||||
core_id[num_workers++] = atoi(curptr);
|
||||
}
|
||||
|
||||
curptr=strtok_r(NULL,",",&saveptr);
|
||||
curptr = strtok_r(NULL, ",", &saveptr);
|
||||
}
|
||||
free(parms_cpy);
|
||||
if (pool->activated && pool->nbThreads==0) {
|
||||
printf("No servers created in the thread pool, exit\n");
|
||||
exit(1);
|
||||
|
||||
if (num_workers) {
|
||||
tpool->q_arr = calloc(num_workers, sizeof(not_q_t));
|
||||
AssertFatal(tpool->q_arr != NULL, "Memory exhausted");
|
||||
|
||||
tpool->t_arr = calloc(num_workers, sizeof(pthread_t));
|
||||
AssertFatal(tpool->t_arr != NULL, "Memory exhausted");
|
||||
}
|
||||
tpool->len_thr = num_workers;
|
||||
|
||||
tpool->index = 0;
|
||||
|
||||
const pthread_barrierattr_t* barrier_attr = NULL;
|
||||
int rc = pthread_barrier_init(&tpool->barrier, barrier_attr, num_workers + 1);
|
||||
DevAssert(rc == 0);
|
||||
|
||||
for (size_t i = 0; i < num_workers; ++i) {
|
||||
task_thread_args_t* args = malloc(sizeof(task_thread_args_t));
|
||||
AssertFatal(args != NULL, "Memory exhausted");
|
||||
args->idx = i;
|
||||
args->tpool = tpool;
|
||||
char name[64];
|
||||
sprintf(name, "%s%ld_%d", tname, i, core_id[i]);
|
||||
threadCreate(&tpool->t_arr[i], worker_thread, args, name, core_id[i], OAI_PRIORITY_RT_MAX);
|
||||
}
|
||||
|
||||
// Syncronize thread pool threads. All the threads started
|
||||
pthread_barrier_wait(&tpool->barrier);
|
||||
}
|
||||
|
||||
void initFloatingCoresTpool(int nbThreads,tpool_t *pool, bool performanceMeas, char *name) {
|
||||
void initFloatingCoresTpool(int nbThreads, tpool_t* pool, bool performanceMeas, char* name)
|
||||
{
|
||||
char threads[1024] = "n";
|
||||
if (nbThreads) {
|
||||
strcpy(threads,"-1");
|
||||
for (int i=1; i < nbThreads; i++)
|
||||
strncat(threads,",-1", sizeof(threads)-1);
|
||||
strcpy(threads, "-1");
|
||||
for (int i = 1; i < nbThreads; i++)
|
||||
strncat(threads, ",-1", sizeof(threads) - 1);
|
||||
}
|
||||
threads[sizeof(threads)-1]=0;
|
||||
threads[sizeof(threads) - 1] = 0;
|
||||
initNamedTpool(threads, pool, performanceMeas, name);
|
||||
}
|
||||
|
||||
|
||||
void abortTpool(tpool_t* tpool)
|
||||
{
|
||||
if (tpool->len_thr > 0) {
|
||||
not_q_t* q_arr = (not_q_t*)tpool->q_arr;
|
||||
|
||||
pushTpool(tpool, (task_t){.args = NULL, .func = NULL});
|
||||
|
||||
for (uint32_t i = 0; i < tpool->len_thr; ++i) {
|
||||
int rc = pthread_join(tpool->t_arr[i], NULL);
|
||||
DevAssert(rc == 0);
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < tpool->len_thr; ++i) {
|
||||
free_not_q(&q_arr[i]);
|
||||
}
|
||||
|
||||
free(tpool->q_arr);
|
||||
free(tpool->t_arr);
|
||||
}
|
||||
|
||||
int rc = pthread_barrier_destroy(&tpool->barrier);
|
||||
DevAssert(rc == 0);
|
||||
}
|
||||
|
||||
@@ -1,26 +1,23 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Author and copyright: Laurent Thomas, open-cells.com
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
#ifndef THREAD_POOL_H
|
||||
#define THREAD_POOL_H
|
||||
@@ -35,316 +32,54 @@
|
||||
#include "assertions.h"
|
||||
#include "common/utils/time_meas.h"
|
||||
#include "common/utils/system.h"
|
||||
#include "task.h"
|
||||
#include "pthread_utils.h"
|
||||
|
||||
#ifdef DEBUG
|
||||
#define THREADINIT PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP
|
||||
#else
|
||||
#define THREADINIT PTHREAD_MUTEX_INITIALIZER
|
||||
#endif
|
||||
#define mutexinit(mutex) {int ret=pthread_mutex_init(&mutex,NULL); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define condinit(signal) {int ret=pthread_cond_init(&signal,NULL); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define mutexlock(mutex) {int ret=pthread_mutex_lock(&mutex); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define mutextrylock(mutex) pthread_mutex_trylock(&mutex)
|
||||
#define mutexunlock(mutex) {int ret=pthread_mutex_unlock(&mutex); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define condwait(condition, mutex) {int ret=pthread_cond_wait(&condition, &mutex); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define condbroadcast(signal) {int ret=pthread_cond_broadcast(&signal); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define condsignal(signal) {int ret=pthread_cond_signal(&signal); \
|
||||
AssertFatal(ret==0,"ret=%d\n",ret);}
|
||||
#define tpool_nbthreads(tpool) (tpool.nbThreads)
|
||||
typedef struct notifiedFIFO_elt_s {
|
||||
struct notifiedFIFO_elt_s *next;
|
||||
uint64_t key; //To filter out elements
|
||||
struct notifiedFIFO_s *reponseFifo;
|
||||
void (*processingFunc)(void *);
|
||||
bool malloced;
|
||||
oai_cputime_t creationTime;
|
||||
oai_cputime_t startProcessingTime;
|
||||
oai_cputime_t endProcessingTime;
|
||||
oai_cputime_t returnTime;
|
||||
// use alignas(32) to align msgData to 32b
|
||||
// user data behind it will be aligned to 32b as well
|
||||
// important! this needs to be the last member in the struct
|
||||
alignas(32) void *msgData;
|
||||
} notifiedFIFO_elt_t;
|
||||
|
||||
typedef struct notifiedFIFO_s {
|
||||
notifiedFIFO_elt_t *outF;
|
||||
notifiedFIFO_elt_t *inF;
|
||||
pthread_mutex_t lockF;
|
||||
pthread_cond_t notifF;
|
||||
bool abortFIFO; // if set, the FIFO always returns NULL -> abort condition
|
||||
} notifiedFIFO_t;
|
||||
typedef struct {
|
||||
pthread_t* t_arr;
|
||||
size_t len_thr;
|
||||
|
||||
// You can use this allocator or use any piece of memory
|
||||
static inline notifiedFIFO_elt_t *newNotifiedFIFO_elt(int size,
|
||||
uint64_t key,
|
||||
notifiedFIFO_t *reponseFifo,
|
||||
void (*processingFunc)(void *)) {
|
||||
notifiedFIFO_elt_t *ret = (notifiedFIFO_elt_t *)memalign(32, sizeof(notifiedFIFO_elt_t) + size);
|
||||
AssertFatal(NULL != ret, "out of memory\n");
|
||||
ret->next=NULL;
|
||||
ret->key=key;
|
||||
ret->reponseFifo=reponseFifo;
|
||||
ret->processingFunc=processingFunc;
|
||||
// We set user data piece aligend 32 bytes to be able to process it with SIMD
|
||||
// msgData is aligned to 32bytes, so everything after will be as well
|
||||
ret->msgData = ((uint8_t *)ret) + sizeof(notifiedFIFO_elt_t);
|
||||
ret->malloced=true;
|
||||
return ret;
|
||||
}
|
||||
_Atomic(uint64_t) index;
|
||||
|
||||
static inline void *NotifiedFifoData(notifiedFIFO_elt_t *elt) {
|
||||
return elt->msgData;
|
||||
}
|
||||
void* q_arr;
|
||||
|
||||
static inline void delNotifiedFIFO_elt(notifiedFIFO_elt_t *elt) {
|
||||
if (elt->malloced) {
|
||||
elt->malloced = false;
|
||||
free(elt);
|
||||
}
|
||||
/* it is allowed to call delNotifiedFIFO_elt when the memory is managed by
|
||||
* the caller */
|
||||
}
|
||||
|
||||
static inline void initNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf) {
|
||||
nf->inF=NULL;
|
||||
nf->outF=NULL;
|
||||
nf->abortFIFO = false;
|
||||
}
|
||||
static inline void initNotifiedFIFO(notifiedFIFO_t *nf) {
|
||||
mutexinit(nf->lockF);
|
||||
condinit (nf->notifF);
|
||||
initNotifiedFIFO_nothreadSafe(nf);
|
||||
// No delete function: the creator has only to free the memory
|
||||
}
|
||||
|
||||
static inline void pushNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg) {
|
||||
msg->next=NULL;
|
||||
|
||||
if (nf->outF == NULL)
|
||||
nf->outF = msg;
|
||||
|
||||
if (nf->inF != NULL)
|
||||
nf->inF->next = msg;
|
||||
|
||||
nf->inF = msg;
|
||||
}
|
||||
|
||||
static inline void pushNotifiedFIFO(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg) {
|
||||
mutexlock(nf->lockF);
|
||||
if (!nf->abortFIFO) {
|
||||
pushNotifiedFIFO_nothreadSafe(nf,msg);
|
||||
condsignal(nf->notifF);
|
||||
}
|
||||
mutexunlock(nf->lockF);
|
||||
}
|
||||
|
||||
static inline notifiedFIFO_elt_t *pullNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf) {
|
||||
if (nf->outF == NULL)
|
||||
return NULL;
|
||||
if (nf->abortFIFO)
|
||||
return NULL;
|
||||
|
||||
notifiedFIFO_elt_t *ret=nf->outF;
|
||||
|
||||
AssertFatal(nf->outF != nf->outF->next,"Circular list in thread pool: push several times the same buffer is forbidden\n");
|
||||
|
||||
nf->outF=nf->outF->next;
|
||||
|
||||
if (nf->outF==NULL)
|
||||
nf->inF=NULL;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline notifiedFIFO_elt_t *pullNotifiedFIFO(notifiedFIFO_t *nf) {
|
||||
mutexlock(nf->lockF);
|
||||
notifiedFIFO_elt_t *ret = NULL;
|
||||
|
||||
while((ret=pullNotifiedFIFO_nothreadSafe(nf)) == NULL && !nf->abortFIFO)
|
||||
condwait(nf->notifF, nf->lockF);
|
||||
|
||||
mutexunlock(nf->lockF);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline notifiedFIFO_elt_t *pollNotifiedFIFO(notifiedFIFO_t *nf) {
|
||||
int tmp=mutextrylock(nf->lockF);
|
||||
|
||||
if (tmp != 0 )
|
||||
return NULL;
|
||||
|
||||
if (nf->abortFIFO) {
|
||||
mutexunlock(nf->lockF);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
notifiedFIFO_elt_t *ret=pullNotifiedFIFO_nothreadSafe(nf);
|
||||
mutexunlock(nf->lockF);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline time_stats_t exec_time_stats_NotifiedFIFO(const notifiedFIFO_elt_t* elt)
|
||||
{
|
||||
time_stats_t ts = {0};
|
||||
if (elt->startProcessingTime == 0 && elt->endProcessingTime == 0)
|
||||
return ts; /* no measurements done */
|
||||
ts.in = elt->startProcessingTime;
|
||||
ts.diff = elt->endProcessingTime - ts.in;
|
||||
ts.p_time = ts.diff;
|
||||
ts.diff_square = ts.diff * ts.diff;
|
||||
ts.max = ts.diff;
|
||||
ts.trials = 1;
|
||||
return ts;
|
||||
}
|
||||
|
||||
// This functions aborts all messages in the queue, and marks the queue as
|
||||
// "aborted", such that every call to it will return NULL
|
||||
static inline void abortNotifiedFIFO(notifiedFIFO_t *nf) {
|
||||
mutexlock(nf->lockF);
|
||||
nf->abortFIFO = true;
|
||||
notifiedFIFO_elt_t **elt = &nf->outF;
|
||||
while(*elt != NULL) {
|
||||
notifiedFIFO_elt_t *p = *elt;
|
||||
*elt = (*elt)->next;
|
||||
delNotifiedFIFO_elt(p);
|
||||
}
|
||||
|
||||
if (nf->outF == NULL)
|
||||
nf->inF = NULL;
|
||||
condbroadcast(nf->notifF);
|
||||
mutexunlock(nf->lockF);
|
||||
}
|
||||
|
||||
struct one_thread {
|
||||
pthread_t threadID;
|
||||
int id;
|
||||
int coreID;
|
||||
char name[256];
|
||||
uint64_t runningOnKey;
|
||||
bool dropJob;
|
||||
bool terminate;
|
||||
struct thread_pool *pool;
|
||||
struct one_thread *next;
|
||||
};
|
||||
|
||||
typedef struct thread_pool {
|
||||
bool activated;
|
||||
bool measurePerf;
|
||||
int traceFd;
|
||||
int dummyKeepReadingTraceFd;
|
||||
uint64_t cpuCyclesMicroSec;
|
||||
int nbThreads;
|
||||
notifiedFIFO_t incomingFifo;
|
||||
struct one_thread *allthreads;
|
||||
pthread_barrier_t barrier;
|
||||
} tpool_t;
|
||||
|
||||
static inline void pushTpool(tpool_t *t, notifiedFIFO_elt_t *msg) {
|
||||
if (t->measurePerf) msg->creationTime=rdtsc_oai();
|
||||
/// @brief Push job to threadpool. May run task inline in case there are no worker threads
|
||||
/// defined for threadpool
|
||||
/// @param tpool threadpool to use
|
||||
/// @param task task description
|
||||
void pushTpool(tpool_t *tpool, task_t task);
|
||||
|
||||
if ( t->activated)
|
||||
pushNotifiedFIFO(&t->incomingFifo, msg);
|
||||
else {
|
||||
if (t->measurePerf)
|
||||
msg->startProcessingTime=rdtsc_oai();
|
||||
/// @brief Abort tpool, stop all threads and return
|
||||
/// @param t
|
||||
void abortTpool(tpool_t *t);
|
||||
|
||||
msg->processingFunc(NotifiedFifoData(msg));
|
||||
|
||||
if (t->measurePerf)
|
||||
msg->endProcessingTime=rdtsc_oai();
|
||||
|
||||
if (msg->reponseFifo)
|
||||
pushNotifiedFIFO(msg->reponseFifo, msg);
|
||||
else
|
||||
delNotifiedFIFO_elt(msg);
|
||||
}
|
||||
}
|
||||
|
||||
static inline notifiedFIFO_elt_t *pullTpool(notifiedFIFO_t *responseFifo, tpool_t *t) {
|
||||
notifiedFIFO_elt_t *msg= pullNotifiedFIFO(responseFifo);
|
||||
if (msg == NULL)
|
||||
return NULL;
|
||||
AssertFatal(t->traceFd != 0, "Thread pool used while not initialized");
|
||||
if (t->measurePerf)
|
||||
msg->returnTime=rdtsc_oai();
|
||||
|
||||
if (t->traceFd > 0) {
|
||||
ssize_t b = write(t->traceFd, msg, sizeof(*msg));
|
||||
AssertFatal(b == sizeof(*msg), "error in write(): %d, %s\n", errno, strerror(errno));
|
||||
}
|
||||
|
||||
return msg;
|
||||
}
|
||||
|
||||
static inline notifiedFIFO_elt_t *tryPullTpool(notifiedFIFO_t *responseFifo, tpool_t *t) {
|
||||
notifiedFIFO_elt_t *msg= pollNotifiedFIFO(responseFifo);
|
||||
AssertFatal(t->traceFd != 0, "Thread pool used while not initialized");
|
||||
if (msg == NULL)
|
||||
return NULL;
|
||||
|
||||
if (t->measurePerf)
|
||||
msg->returnTime=rdtsc_oai();
|
||||
|
||||
if (t->traceFd > 0) {
|
||||
ssize_t b = write(t->traceFd, msg, sizeof(*msg));
|
||||
AssertFatal(b == sizeof(*msg), "error in write(): %d, %s\n", errno, strerror(errno));
|
||||
}
|
||||
|
||||
return msg;
|
||||
}
|
||||
|
||||
static inline int abortTpool(tpool_t *t) {
|
||||
int nbRemoved=0;
|
||||
/* disables threading: if a message comes in now, we cannot have a race below
|
||||
* as each thread will simply execute the message itself */
|
||||
t->activated = false;
|
||||
notifiedFIFO_t *nf=&t->incomingFifo;
|
||||
mutexlock(nf->lockF);
|
||||
nf->abortFIFO = true;
|
||||
notifiedFIFO_elt_t **start=&nf->outF;
|
||||
|
||||
/* mark threads to abort them */
|
||||
struct one_thread *thread = t->allthreads;
|
||||
while (thread != NULL) {
|
||||
thread->dropJob = true;
|
||||
thread->terminate = true;
|
||||
nbRemoved++;
|
||||
thread = thread->next;
|
||||
}
|
||||
|
||||
/* clear FIFOs */
|
||||
while(*start!=NULL) {
|
||||
notifiedFIFO_elt_t **request=start;
|
||||
*start=(*start)->next;
|
||||
delNotifiedFIFO_elt(*request);
|
||||
*request = NULL;
|
||||
nbRemoved++;
|
||||
}
|
||||
|
||||
if (t->incomingFifo.outF==NULL)
|
||||
t->incomingFifo.inF=NULL;
|
||||
|
||||
condbroadcast(t->incomingFifo.notifF);
|
||||
mutexunlock(nf->lockF);
|
||||
|
||||
/* join threads that are still runing */
|
||||
thread = t->allthreads;
|
||||
while (thread != NULL) {
|
||||
pthread_cancel(thread->threadID);
|
||||
struct one_thread *next = thread->next;
|
||||
free(thread);
|
||||
thread = next;
|
||||
}
|
||||
|
||||
return nbRemoved;
|
||||
}
|
||||
/// @brief Initialize a threadPool.
|
||||
/// @param params A string in a form of "<int>,<int>,<int>,...,<int>" or "n"
|
||||
/// if params is "n" threadpool is disabled and all functions are called inline.
|
||||
/// else if params is a list of comma separated integers, the number of threads
|
||||
/// is equal to the list length and each thread is pinned to a core indicated by
|
||||
/// the integer value. If -1 is specified thread is not pinned to any core.
|
||||
/// @param pool Theadpool
|
||||
/// @param performanceMeas unused
|
||||
/// @param name Name of the theadpool, will be used as pthread names for the pool worker threads
|
||||
void initNamedTpool(char *params,tpool_t *pool, bool performanceMeas, char *name);
|
||||
|
||||
/// @brief Initialize a threadpool with floating core worker threads only
|
||||
/// @param nbThreads number of floating core worker threads
|
||||
/// @param pool Threadpool
|
||||
/// @param performanceMeas unused
|
||||
/// @param name Name of the theadpool, will be used as pthread names for the pool worker threads
|
||||
void initFloatingCoresTpool(int nbThreads,tpool_t *pool, bool performanceMeas, char *name);
|
||||
|
||||
/// Convenience macro
|
||||
#define initTpool(PARAMPTR,TPOOLPTR, MEASURFLAG) initNamedTpool(PARAMPTR,TPOOLPTR, MEASURFLAG, NULL)
|
||||
#endif
|
||||
|
||||
@@ -1,198 +1,7 @@
|
||||
# Thread pool
|
||||
|
||||
The **thread pool** is a working server, made of a set of worker threads that can be mapped on CPU cores. Each thread pool has an **input queue** ("**FIFO**"), from which its workers pick **jobs** (FIFO element) to execute. When a job is done, the worker sends a message to an output queue, if it has been defined.
|
||||
The **thread pool** is a working server, made of a set of worker threads that can be mapped on CPU cores.
|
||||
|
||||
All the thread pool functions are thread safe. The functions executed by worker threads are provided by the thread pool client, so the client has to handle the concurrency/parallel execution of his functions.
|
||||
|
||||
## license
|
||||
Author:
|
||||
Laurent Thomas, Open cells project
|
||||
The owner share this piece code to Openairsoftware alliance as per OSA license terms
|
||||
With contributions from Robert Schmidt, OSA
|
||||
|
||||
# Jobs
|
||||
|
||||
A job is a message of type `notifiedFIFO_elt_t`:
|
||||
|
||||
* `next`: internal FIFO chain, do not set it
|
||||
* `key`: a `long int` that the client can use to identify a job or a group of messages
|
||||
* `ResponseFifo`: if the client defines a response FIFO, the job will be posted back after processing
|
||||
* `processingFunc`: any function (of type `void processingFunc(void *)`) that a worker will process
|
||||
* `msgData`: the data passed to `processingFunc`. It can be added automatically, or you can set it to a buffer you are managing
|
||||
* `malloced`: a boolean that enables internal free in the case of no return FIFO or abort feature
|
||||
|
||||
The job messages can be created and destroyed with `newNotifiedFIFO_elt()` and `delNotifiedFIFO_elt()`, or managed by the client:
|
||||
|
||||
* `newNotifiedFIFO_elt()`: Creates a job, that will later be used in queues/FIFO
|
||||
* `delNotifiedFIFO_elt()`: Deletes a job
|
||||
* `NotifiedFifoData()`: gives a pointer to the beginning of free usage memory in a job (you can put any data there, up to 'size' parameter you passed to `newNotifiedFIFO_elt()`)
|
||||
|
||||
These 3 calls are not mandatory, you can also use your own memory to save the `malloc()`/`free()` that are behind these calls.
|
||||
|
||||
## API details
|
||||
|
||||
### `notifiedFIFO_elt_t *newNotifiedFIFO_elt(int size, uint64_t key, notifiedFIFO_t *reponseFifo, void (*processingFunc)(void *))`
|
||||
|
||||
Creates a new job. The data part of the job will have size `size`, the job can be identified via `key`. `reponseFifo` is a FIFO queue to which the job should be pushed after being processed by a thread pool (see below for details) using function `processingFunc()`.
|
||||
|
||||
The function returns a pointer to an allocated job.
|
||||
|
||||
### `void delNotifiedFIFO_elt(notifiedFIFO_elt_t *elt)`
|
||||
|
||||
Free the memory allocated for a job `elt`.
|
||||
|
||||
### `void *NotifiedFifoData(notifiedFIFO_elt_t *elt)`
|
||||
|
||||
Returns a pointer (`void *`) to the data segment of an allocated job.
|
||||
|
||||
# Queues of jobs (`FIFO`)
|
||||
|
||||
Queues can be used to enqueue messages/jobs, of type `notifiedFIFO_t`.
|
||||
|
||||
* `initNotifiedFIFO()`: Create a queue
|
||||
* No delete function is required, the creator has only to free the data of type `notifiedFIFO_t`
|
||||
* `pushNotifiedFIFO()`: Add a job to a queue
|
||||
* `pullNotifiedFIFO()`: Pull a job from a queue. This call is blocking until a job arrived.
|
||||
* `pollNotifiedFIFO()`: Pull a job from a queue. This call is not blocking, so it returns always very shortly
|
||||
* `abortNotifiedFIFO()`: Aborts a FIFO, such that it will always return `NULL`
|
||||
|
||||
Note that in 99.9% of cases, `pull()` is better than `poll()`.
|
||||
|
||||
The above push/pull/poll functions are built on not-thread-safe versions of these functions that are described below.
|
||||
|
||||
## Low level: fast and simple, but not thread-safe
|
||||
|
||||
* `initNotifiedFIFO_nothreadSafe()`: Create a queue
|
||||
* `pushNotifiedFIFO_nothreadSafe()`: Add a element in a queue
|
||||
* `pullNotifiedFIFO_nothreadSafe()`: Pull a element from a queue
|
||||
|
||||
As these queues are not thread safe, there is NO blocking mechanism, neither `pull()` versus `poll()` calls
|
||||
|
||||
There is no delete for a message queue: you only have to abandon the memory you allocated to call `initNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf)`
|
||||
|
||||
If you malloced the memory under `nf` parameter, you have to free it.
|
||||
|
||||
## FIFO queue abort
|
||||
|
||||
It is possible to *abort* the FIFO queue. In this case, any pulling/polling of jobs in the queue will return `NULL`. This can be used when stopping a program, to release multiple queues that might depend on each other (a proper design would try to avoid such dependencies, though).
|
||||
|
||||
## API details
|
||||
|
||||
### `void initNotifiedFIFO(notifiedFIFO_t *nf)`
|
||||
### `void initNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf)`
|
||||
|
||||
Initializes a FIFO queue pointed to by `nf` ("notified FIFO").
|
||||
|
||||
### `void pushNotifiedFIFO(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg)`
|
||||
### `void pushNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg)`
|
||||
|
||||
Pushes a job `msg` to the FIFO queue `nf`.
|
||||
|
||||
### `notifiedFIFO_elt_t *pullNotifiedFIFO(notifiedFIFO_t *nf)`
|
||||
|
||||
Pulls a job from a FIFO queue `nf`. The returned pointer points to the job. This call is blocking, i.e., a caller will be blocked until there is a message.
|
||||
|
||||
If the FIFO has been aborted, the returned pointer will be `NULL`. In other words, if the call to this functions returns `NULL`, the FIFO has been deactivated.
|
||||
|
||||
### `notifiedFIFO_elt_t *pollNotifiedFIFO(notifiedFIFO_t *nf)`
|
||||
### `notifiedFIFO_elt_t *pullNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf)`
|
||||
|
||||
Like `pullNotifiedFIFO()`, but non-blocking: they check if the queue `nf` contains a job, and return it. Otherwise, they return `NULL`. Note the slight difference in naming: the thread-safe function is called *poll*, not *pull*.
|
||||
|
||||
Note that unlike for `pullNotifiedFIFO()`, returning `NULL` does not inform whether the queue has been aborted; the caller should manually check the `abortFIFO` flag of `nf` in this case.
|
||||
|
||||
### `void abortNotifiedFIFO(notifiedFIFO_t *nf)`
|
||||
|
||||
Aborts the entire FIFO queue `nf`: all jobs will be dropped, and the FIFO is marked as aborted, such that a call to `pullNotifiedFIFO()` returns `NULL`.
|
||||
|
||||
Returns the total number of jobs that were waiting or under execution.
|
||||
|
||||
# Thread pools
|
||||
|
||||
## Initialization
|
||||
|
||||
The clients can create one or more thread pools with
|
||||
|
||||
* `initTpool(char *params, tpool_t *pool, bool performanceMeas)` or
|
||||
* `initNamedTpool(char *params,tpool_t *pool, bool performanceMeas , char *name)`
|
||||
|
||||
The threads are governed by the Linux real time scheduler, and their name is set automatically to `Tpool<thread index>_<core id>` if `initTpool()` is used or to `<name><thread index>_<core id>` when `initNamedTpool()` is used.
|
||||
|
||||
## Adding jobs
|
||||
|
||||
The client create their jobs messages as a `notifiedFIFO_elt_t`. They then push it with `pushTpool()` which internally calls `push_notifiedFIFO()`.
|
||||
|
||||
If they need a return value (e.g., result of a computation), they have to create response queues with `init_notifiedFIFO()` and set this FIFO pointer in the `notifiedFIFO_elt_t` before pushing the job. This way, multiple result queues can be used in one thread pool.
|
||||
|
||||
## Abort
|
||||
|
||||
`abortTpool()` kills all jobs in the Tpool, and terminates the pool.
|
||||
|
||||
## API details
|
||||
|
||||
Each thread pool (there can be several in the same process) should be initialized once using one of the two API's:
|
||||
|
||||
### `void initNamedTpool(char *params, tpool_t *pool, bool performanceMeas, char *name)`
|
||||
|
||||
### `void initTpool(char *params, tpool_t *pool, bool performanceMeas)`
|
||||
|
||||
|
||||
The `params` string describes a list of cores, separated by "," that run a worker thread. If the core exists on the CPU, the thread pool initialization sets the affinity between this thread and the related core. The following options are available:
|
||||
|
||||
* `N`: if an `N` is in the parameter list, no threads are created, and the threadpool will be disabled (jobs will be treated by each calling thread independently).
|
||||
* A number that represent a valid CPU core on the target CPU, i.e., if there are `M` cores, a number within `[0,M-1]`: A thread is created and pinned to the core (with set affinity)
|
||||
* `-1`: The thread will not be mapped onto a specific core (a "floating" thread)
|
||||
|
||||
Example: `-1,-1,-1`: the thread pool is made of 3 floating threads.
|
||||
Be careful with a fixed allocation: it is hard to be more clever than Linux kernel!
|
||||
|
||||
`pool` is a pointer to the thread pool object.
|
||||
|
||||
`performanceMeas` is a flag to enable measurements (see also below).
|
||||
|
||||
`name` is used to build the thread names.
|
||||
|
||||
### `void pushTpool(tpool_t *t, notifiedFIFO_elt_t *msg)`
|
||||
|
||||
Adds a job for processing in the thread pool.
|
||||
|
||||
The job data you can set are, inside `msg`:
|
||||
|
||||
* `key`: an arbitrary key to find a job in a response queue.
|
||||
* `reponseFifo`: if non-`NULL`, the message will be sent back on this queue when the job is done. If `NULL`, the thread pool automatically frees the job when it is done.
|
||||
* `processingFunc`: the function to execute for this job.
|
||||
|
||||
The `processingFunc` prototype is `void <func>(void *memory)`. The data part of the job (the pointer returned by `NotifiedFifoData(msg)`) is passed to the function. The job data will be updated by `processingFunc`, and the job will be pushed into return queue (the parameter `reponseFifo`).
|
||||
|
||||
### `notifiedFIFO_elt_t *pullTpool(notifiedFIFO_t *responseFifo, tpool_t *t)`
|
||||
|
||||
Collects a job result in a return queue that has been defined in `msg` when calling `pushTpool()`, and that has been updated by `processingFunc()`. Returns the corresponding job pointer of type `notifiedFIFO_elt_t *`.
|
||||
|
||||
Multiple return queues might be useful. Consider the following example in the eNB: I created one single thread pool (because it depends mainly on CPU hardware), but i use two return queues: one for turbo encode, one for turbo decode. Thus, when performing Turbo encoding/decoding, jobs are pushed into a single thread pool, but will end up in different result queues, such that DL/UL processing can be separated more easily.
|
||||
|
||||
### `notifiedFIFO_elt_t *tryPullTpool(notifiedFIFO_t *responseFifo, tpool_t *t)`
|
||||
|
||||
The same as `pullTpool()` in a non-blocking fashion (an alternative name would have been `pollTpool()`).
|
||||
|
||||
### `int abortTpool(tpool_t *t)`
|
||||
|
||||
Aborts the complete Tpool: cancel every work in the input queue, marks to drop existing jobs in processing, and terminates all worker threads. It is afterwards still possible to call functions such as `pushTpool()`, but each calling thread will execute the job itself.
|
||||
|
||||
Returns the total number of jobs that were aborted, i.e., waiting for execution or being executed.
|
||||
|
||||
## Performance measurements
|
||||
|
||||
A performance measurement is integrated: the pool will automacillay fill timestamps if you set the environement variable `OAI_THREADPOOLMEASUREMENTS` to a valid file name. The following measurements will be written to Linux pipe on a per-job basis:
|
||||
|
||||
* `creationTime`: time the request is push to the pool;
|
||||
* `startProcessingTime`: time a worker start to run on the job
|
||||
* `endProcessingTime`: time the worker finished the job
|
||||
* `returnTime`: time the client reads the result
|
||||
|
||||
The `measurement_display` tool to read the Linux pipe and display it in ASCII is provided.
|
||||
In the cmake build directory, type `make/ninja measurement_display`. Use as
|
||||
follows:
|
||||
```
|
||||
sudo OAI_THREADPOOLMEASUREMENTS=tpool.meas ./nr-softmodem ...
|
||||
./measurement_display tpool.meas
|
||||
```
|
||||
See header file doxygen comments for detailed description of the API
|
||||
|
||||
Reference in New Issue
Block a user