diff --git a/.gitmodules b/.gitmodules index 0f50366a4d..fea766a4e4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "src/components/lib/ck/ck"] path = src/components/lib/ck/ck url = https://github.com/gwsystems/ck.git +[submodule "src/components/lib/linux"] + path = src/components/lib/linux + url = https://github.com/lkl/linux.git diff --git a/src/components/implementation/tests/crt_tests/Makefile b/src/components/implementation/tests/crt_tests/Makefile new file mode 100644 index 0000000000..1469929f49 --- /dev/null +++ b/src/components/implementation/tests/crt_tests/Makefile @@ -0,0 +1,8 @@ +COMPONENT=crtt.o +INTERFACES= +DEPENDENCIES= +IF_LIB= +ADDITIONAL_LIBS=-lcobj_format $(LIBSLRAW) -lsl_mod_fprr -lsl_thd_static_backend -lsl_blkpt + +include ../../Makefile.subsubdir +MANDITORY_LIB=simple_stklib.o diff --git a/src/components/implementation/tests/crt_tests/README.md b/src/components/implementation/tests/crt_tests/README.md new file mode 100644 index 0000000000..71793119b3 --- /dev/null +++ b/src/components/implementation/tests/crt_tests/README.md @@ -0,0 +1,128 @@ +# CRT Tests + +## General Description of crt blkpt +``` +The event count/block point is an abstraction to synchronize the + blocking behavior of different threads on abstract events. The + events are usually tied to a specific state of another + data-structure (into which the blkpt is embedded). For example, a + lock is taken and released thus generating an event for any + blocking threads, or a ring buffer has a data item inserted into + it, thus generating an event for any threads waiting for + data. Concretely, we want a number of threads to be able to block, + and a thread to be able to wake up one, or all of them. The + challenge is solving a single race-condition: + + thd 0: check data-structure, determine the need for blocking and + waiting for an event + thd 0: preemption, switching to thd 1 + thd 1: check data-structure, determine that an event is generated + thd 1: call the scheduler, and wake all blocked threads (not + including thd 0 yet) + thd 1: preempt, and switch to thd 0 + thd 0: call scheduler to block + + The resulting state is that thd 1 should have unblocked thd 0, but + due to a race, the thd 0 will be blocked awaiting the next event + that may never come. Event counts are meant to solve this + problem. Traditional systems solve this problem using condition + variables and a lock around the scheduling logic, but if you want + to decouple the data-structure from the scheduler (e.g. as they are + in different modes, or components), this is a fundamental problem. + + The event count abstraction: + + Assume the data-structure generating events has at least three + states: + S0: available + S1: unavailable + S2: unavailable & subscribed + + The transitions within the data-structure are: + {S0->S1, S1->S0, S1->S2, S2->S0} + + Every transition into S0 is an abstract event . Threads that look + at the state of the data-structure, and must block waiting for its + state to change, wait for such an event to wakeup. + + The data-structure must define its own mapping to this state + machine. A few examples: + + Mutexes: + S0: Not locked. + S1: Locked and held by thread 0. + S2: Locked and held by thread 0, and threads 1...N contend the lock + + Ring buffer (for simplicity, assuming it never fills): + S0: data items in ring buffer + S1: no data in ring buffer + S2: no data in ring buffer, and thread(s) are waiting for data + + The event counts are used to track the threads that use the + data-structure when transitioning from S1->S2 (block thread), when + it is in S2 (block additional threads), and when it transitions + from S2->S0 (wakeup blocked threads). + + The event count is used in the following way: + + S0->S1: + data-structure (DS) operation + E.g. not locked -> locked, or + dequeue from ring with single data item + + S1->S0: + blkpt_checkpoint(ec) (not used) + data-structure (DS) operation + assert(blkpt_has_blocked(ec) == false) (as we're in S1) + blkpt_trigger(ec) (won't do much as noone is blocked) + E.g. unlock with no contention, or + enqueue with no dequeuing threads + + S1->S2: + cp = blkpt_checkpoint(ec) + data-structure (DS) operation, determine we need to await event + blkpt_wait(ec, cp) + retry (this is why event counts can be used with lock-free data-structs) + E.g. locked -> contended + empty ring -> waiting for data + + S2->S0: + data-structure (DS) operation + assert(blkpt_has_blocked(ec) == true) (as we're in S2) + blkpt_trigger(ec) (wake blocked threads!) + E.g. unlock with contention, or + enqueue with dequeuing threads + + Event count optimization : + + We prevent the race above using an epoch (count) for the events + thus the name. However, to avoid rapid wraparound on the epoch, we + only increment the epoch when the race condition is possible. That + is to say, we only increment the event count when the + data-structure has blocked threads. This not only delays + wraparound, it also will avoid an atomic instruction for all + operations that don't involve blocked threads (a common-case, + exemplified by futexes, for example). + + Usage optimization: + + Because of the event counter optimization to only use expensive + operations when triggering there are blocked threads, the user of + this API can trigger whenever transitioning back to S0. + ``` + +## Tests to implement + +- [ ] Basic Premption Check + +We will have a thread preempt another thread using thd_yield(). + +Pseudocode +``` +- set up a checkpoint on a thread +- preempt said thread using thd_yield() +- wake up all thds +- switch to original thd +- block thd (no one will wake this thd up) + +``` diff --git a/src/components/implementation/tests/crt_tests/crttests.c b/src/components/implementation/tests/crt_tests/crttests.c new file mode 100644 index 0000000000..ab735a62de --- /dev/null +++ b/src/components/implementation/tests/crt_tests/crttests.c @@ -0,0 +1,424 @@ +/* + * Copyright 2016, Phani Gadepalli and Gabriel Parmer, GWU, gparmer@gwu.edu. + * + * This uses a two clause BSD License. + */ + +#include +#include +#include +#include + +#include +#include + +struct cos_compinfo *ci; + +#define CHAN_ITER 1000000 +#define NCHANTHDS 5 +#define CHAN_BATCH 3 + + +CRT_CHAN_STATIC_ALLOC(c0, int, 4); +CRT_CHAN_STATIC_ALLOC(c1, int, 4); +CRT_CHAN_STATIC_ALLOC(c2, int, 4); +CRT_CHAN_STATIC_ALLOC(c3, int, 4); +CRT_CHAN_STATIC_ALLOC(c4, int, 4); + +CRT_CHAN_TYPE_PROTOTYPES(test, int, 4); +struct crt_chan *chans[NCHANTHDS + 1]; +struct sl_thd * chan_thds[NCHANTHDS] = { + NULL, +}; + +typedef enum +{ + CHILLING = 0, + RECVING, + SENDING +} actions_t; +unsigned long status[NCHANTHDS]; +unsigned long cnts[NCHANTHDS] = { + 0, +}; + +int +chantest_is_deadlocked(void) +{ + int i; + actions_t s = status[0]; + + /* Are all threads in the same blocked state? */ + for (i = 0; i < NCHANTHDS; i++) { + if (status[i] == CHILLING || status[i] != s) return 0; + } + + return 1; +} + +void +chantest_send(int thd_off, struct crt_chan *c) +{ + int send = cos_thdid(); + + if (crt_chan_full_test(c)) status[thd_off] = SENDING; + if (!chantest_is_deadlocked()) { + /* printc("\t%d: send\n", cos_thdid()); */ + crt_chan_send_test(c, &send); + } + status[thd_off] = CHILLING; +} + +void +chantest_recv(int thd_off, struct crt_chan *c) +{ + int recv; + + if (crt_chan_empty_test(c)) status[thd_off] = RECVING; + if (!chantest_is_deadlocked()) { + /* printc("\t%d: recv\n", cos_thdid()); */ + crt_chan_recv_test(c, &recv); + cnts[thd_off]++; + } + status[thd_off] = CHILLING; +} + +void +chan_thd(void *d) +{ + int thd_off = (int)d; + struct crt_chan **chan_pair = &chans[thd_off]; + int recv; + int i; + + for (i = 0; i < CHAN_ITER; i++) { + int j; + + /* printc("%d: pre-send\n", cos_thdid()); */ + for (j = 0; j < CHAN_BATCH; j++) { chantest_send(thd_off, chan_pair[1]); } + + /* printc("%d: pre-recv\n", cos_thdid()); */ + for (j = 0; j < CHAN_BATCH; j++) { chantest_recv(thd_off, chan_pair[0]); } + } + + printc("SUCCESS! Counts (should be within %d of each other): ", NCHANTHDS * CHAN_BATCH); + for (i = 0; i < NCHANTHDS; i++) { printc("\t%ld", cnts[i]); } + printc("\n"); + while (1) + ; +} + +void +idle_thd(void *d) +{ + printc("FAILURE: deadlock!\n"); + while (1) + ; +} + +void +test_chan(void) +{ + int i; + struct sl_thd * idle; + union sched_param_union idle_param = {.c = {.type = SCHEDP_PRIO, .value = 10}}; + + union sched_param_union sps[] = {{.c = {.type = SCHEDP_PRIO, .value = 7}}, + {.c = {.type = SCHEDP_PRIO, .value = 6}}, + {.c = {.type = SCHEDP_PRIO, .value = 8}}, + {.c = {.type = SCHEDP_PRIO, .value = 5}}, + {.c = {.type = SCHEDP_PRIO, .value = 5}}}; + + chans[0] = c0; + chans[1] = c1; + chans[2] = c2; + chans[3] = c3; + chans[4] = c4; + chans[5] = c0; + + for (i = 0; i < NCHANTHDS; i++) { crt_chan_init_test(chans[i]); } + + printc("Create threads:\n"); + for (i = 0; i < NCHANTHDS; i++) { + chan_thds[i] = sl_thd_alloc(chan_thd, (void *)i); + assert(chan_thds[i]); + printc("\tcreating thread %d at prio %d\n", sl_thd_thdid(chan_thds[i]), sps[i].c.value); + sl_thd_param_set(chan_thds[i], sps[i].v); + } + idle = sl_thd_alloc(idle_thd, NULL); + printc("\tcreating IDLE %d at prio %d\n", sl_thd_thdid(idle), idle_param.c.value); + sl_thd_param_set(idle, idle_param.v); +} + +#define LOCK_ITER 1000000 +#define NLOCKTHDS 4 +struct crt_lock lock; +struct sl_thd * lock_thds[NLOCKTHDS] = { + NULL, +}; +unsigned int progress[NLOCKTHDS] = { + 0, +}; +volatile thdid_t holder; + +thdid_t +next_thd(void) +{ + return sl_thd_thdid(lock_thds[(unsigned int)(ps_tsc() % NLOCKTHDS)]); +} + +void +lock_thd(void *d) +{ + int i, cnt, me = -1; + + for (i = 0; i < NLOCKTHDS; i++) { + if (sl_thd_thdid(lock_thds[i]) != cos_thdid()) continue; + + me = i; + } + assert(me != -1); + + sl_thd_yield(sl_thd_thdid(lock_thds[1])); + + for (i = 0; i < LOCK_ITER; i++) { + crt_lock_take(&lock); + + progress[me]++; + holder = cos_thdid(); + + sl_thd_yield(next_thd()); + + if (holder != cos_thdid()) { + printc("FAILURE\n"); + BUG(); + } + crt_lock_release(&lock); + sl_thd_yield(next_thd()); + } + + for (i = 0; i < NLOCKTHDS; i++) { + if (i == me) continue; + + if (progress[i] < LOCK_ITER) { sl_thd_yield(sl_thd_thdid(lock_thds[i])); } + } + + printc("SUCCESS!"); + while (1) + ; +} + +void +test_lock(void) +{ + int i; + union sched_param_union sps[] = {{.c = {.type = SCHEDP_PRIO, .value = 5}}, + {.c = {.type = SCHEDP_PRIO, .value = 6}}, + {.c = {.type = SCHEDP_PRIO, .value = 6}}, + {.c = {.type = SCHEDP_PRIO, .value = 7}}}; + + crt_lock_init(&lock); + + printc("Create threads:\n"); + for (i = 0; i < NLOCKTHDS; i++) { + lock_thds[i] = sl_thd_alloc(lock_thd, NULL); + printc("\tcreating thread %d at prio %d\n", sl_thd_thdid(lock_thds[i]), sps[i].c.value); + sl_thd_param_set(lock_thds[i], sps[i].v); + } +} + +struct sl_thd *blk_thds1[1] = { + NULL, +}; + +struct sl_thd *blk_thds2[2] = { + NULL, +}; + +struct sl_thd *blk_thds3[2] = { + NULL, +}; + + + +int progress_shared = 0; + +struct crt_blkpt blkpt_test_1; +struct crt_blkpt blkpt_test_2; +struct crt_blkpt blkpt_test_3; + + +void +blk_thd1(void *d) +{ + + + int i, cnt, me = -1; + //put this into function + for (i = 0; i < 1; i++) { + if (sl_thd_thdid(blk_thds1[i]) != cos_thdid()) continue; + + me = i; + } + assert(me != -1); + + struct crt_blkpt_checkpoint chkpt; + + /* set up a checkoint */ + crt_blkpt_checkpoint(&blkpt_test_1, &chkpt); + + /* wake up all threads */ + crt_blkpt_trigger(&blkpt_test_1, 0); + + //check API behavior in code + + /* there is no thread to preempt us so these should be the same */ + if(blkpt_test_1.epoch_blocked != chkpt.epoch_blocked) { + printc("FAILURE"); + while(1) + ; + } + + printc("SUCCESS!"); + while (1) + ; +} +void +blk_thd2(void *d) +{ + struct crt_blkpt_checkpoint chkpt; + + /* set up a checkoint */ + crt_blkpt_checkpoint(&blkpt_test_2, &chkpt); + + crt_blkpt_trigger(&blkpt_test_2, 0); + + crt_blkpt_wait(&blkpt_test_2, 0, &chkpt); + + progress_shared++; + + sl_thd_yield(sl_thd_thdid(blk_thds2[1])); + + while(1); +} + +void +blk_thd3(void *d) +{ + if(progress_shared == 0) { + printc("Failure"); + while(1); + } + printc("Success!"); + while(1); +} + +void +blk_thd4(void *d) +{ + struct crt_blkpt_checkpoint chkpt; + + crt_blkpt_checkpoint(&blkpt_test_3, &chkpt); + + crt_blkpt_wait(&blkpt_test_3, 0, &chkpt); + + progress_shared++; + + sl_thd_yield(sl_thd_thdid(blk_thds3[1])); + + while(1); + +} + +void +blk_thd5(void *d) +{ + if(progress_shared == 0) { + printc("Success!"); + while(1); + } + printc("Failure"); + while(1); +} + + + +void +test_blkpt1(void) +{ + int i; + union sched_param_union sps[] = {{.c = {.type = SCHEDP_PRIO, .value = 5}}}; + crt_blkpt_init(&blkpt_test_1); + printc("Create thread:\n"); + blk_thds1[0] = sl_thd_alloc(blk_thd1, NULL); + printc("\tcreating thread %d at prio %d\n", sl_thd_thdid(blk_thds1[0]), sps[0].c.value); + sl_thd_param_set(blk_thds1[0], sps[0].v); + + +} + +void +test_blkpt2(void) +{ + int i; + union sched_param_union sps[] = {{.c = {.type = SCHEDP_PRIO, .value = 6}}, + {.c = {.type = SCHEDP_PRIO, .value = 7}}}; + + progress_shared = 0; + crt_blkpt_init(&blkpt_test_2); + printc("Create thread:\n"); + blk_thds2[0] = sl_thd_alloc(blk_thd2, NULL); + blk_thds2[1] = sl_thd_alloc(blk_thd3, NULL); + printc("\tcreating thread %d at prio %d\n", sl_thd_thdid(blk_thds2[0]), sps[0].c.value); + printc("\tcreating thread %d at prio %d\n", sl_thd_thdid(blk_thds2[1]), sps[1].c.value); + sl_thd_param_set(blk_thds2[0], sps[0].v); + sl_thd_param_set(blk_thds2[1], sps[1].v); + +} + +void +test_blkpt3(void) +{ + int i; + union sched_param_union sps[] = {{.c = {.type = SCHEDP_PRIO, .value = 6}}, + {.c = {.type = SCHEDP_PRIO, .value = 7}}}; + + progress_shared = 0; + crt_blkpt_init(&blkpt_test_3); + printc("Create thread:\n"); + blk_thds3[0] = sl_thd_alloc(blk_thd4, NULL); + blk_thds3[1] = sl_thd_alloc(blk_thd5, NULL); + printc("\tcreating thread %d at prio %d\n", sl_thd_thdid(blk_thds3[0]), sps[0].c.value); + printc("\tcreating thread %d at prio %d\n", sl_thd_thdid(blk_thds3[1]), sps[1].c.value); + sl_thd_param_set(blk_thds3[0], sps[0].v); + sl_thd_param_set(blk_thds3[1], sps[1].v); + +} + + + +void +cos_init(void) +{ + struct cos_defcompinfo *defci = cos_defcompinfo_curr_get(); + ci = cos_compinfo_get(defci); + + printc("Unit-test for the crt (sl)\n"); + cos_meminfo_init(&(ci->mi), BOOT_MEM_KM_BASE, COS_MEM_KERN_PA_SZ, BOOT_CAPTBL_SELF_UNTYPED_PT); + cos_defcompinfo_init(); + sl_init(SL_MIN_PERIOD_US); + + // test_lock(); + // test_chan(); + + // test_blkpt1(); + // test_blkpt2(); + test_blkpt3(); + + printc("Running benchmark...\n"); + sl_sched_loop_nonblock(); + + assert(0); + + return; +} diff --git a/src/components/include/cos_lock.h b/src/components/include/cos_lock.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/components/include/crt_blkpt.h b/src/components/include/crt_blkpt.h new file mode 100644 index 0000000000..c29e4943ee --- /dev/null +++ b/src/components/include/crt_blkpt.h @@ -0,0 +1,299 @@ +#ifndef CRT_BLKPT_H +#define CRT_BLKPT_H + +#include +#include +#include + +/*** + * The event count/block point is an abstraction to synchronize the + * blocking behavior of different threads on abstract events. The + * events are usually tied to a specific state of another + * data-structure (into which the blkpt is embedded). For example, a + * lock is taken and released thus generating an event for any + * blocking threads, or a ring buffer has a data item inserted into + * it, thus generating an event for any threads waiting for + * data. Concretely, we want a number of threads to be able to block, + * and a thread to be able to wake up one, or all of them. The + * challenge is solving a single race-condition: + * + * thd 0: check data-structure, determine the need for blocking and + * waiting for an event + * thd 0: preemption, switching to thd 1 + * thd 1: check data-structure, determine that an event is generated + * thd 1: call the scheduler, and wake all blocked threads (not + * including thd 0 yet) + * thd 1: preempt, and switch to thd 0 + * thd 0: call scheduler to block + * + * The resulting state is that thd 1 should have unblocked thd 0, but + * due to a race, the thd 0 will be blocked awaiting the *next* event + * that may never come. Event counts are meant to solve this + * problem. Traditional systems solve this problem using condition + * variables and a lock around the scheduling logic, but if you want + * to decouple the data-structure from the scheduler (e.g. as they are + * in different modes, or components), this is a fundamental problem. + * + * The event count abstraction: + * + * Assume the data-structure generating events has at least three + * states: + * S0: available + * S1: unavailable + * S2: unavailable & subscribed + * + * The transitions within the data-structure are: + * {S0->S1, S1->S0, S1->S2, S2->S0} + * + * Every transition into S0 is an abstract *event*. Threads that look + * at the state of the data-structure, and must block waiting for its + * state to change, wait for such an event to wakeup. + * + * The data-structure must define its own mapping to this state + * machine. A few examples: + * + * Mutexes: + * S0: Not locked. + * S1: Locked and held by thread 0. + * S2: Locked and held by thread 0, and threads 1...N contend the lock + * + * Ring buffer (for simplicity, assuming it never fills): + * S0: data items in ring buffer + * S1: no data in ring buffer + * S2: no data in ring buffer, and thread(s) are waiting for data + * + * The event counts are used to track the threads that use the + * data-structure when transitioning from S1->S2 (block thread), when + * it is in S2 (block additional threads), and when it transitions + * from S2->S0 (wakeup blocked threads). + * + * The event count is used in the following way: + * + * S0->S1: + * data-structure (DS) operation + * E.g. not locked -> locked, or + * dequeue from ring with single data item + * + * S1->S0: + * blkpt_checkpoint(ec) (not used) + * data-structure (DS) operation + * assert(blkpt_has_blocked(ec) == false) (as we're in S1) + * blkpt_trigger(ec) (won't do much as noone is blocked) + * E.g. unlock with no contention, or + * enqueue with no dequeuing threads + * + * S1->S2: + * cp = blkpt_checkpoint(ec) + * data-structure (DS) operation, determine we need to await event + * blkpt_wait(ec, cp) + * retry (this is why event counts can be used with lock-free data-structs) + * E.g. locked -> contended + * empty ring -> waiting for data + * + * S2->S0: + * data-structure (DS) operation + * assert(blkpt_has_blocked(ec) == true) (as we're in S2) + * blkpt_trigger(ec) (wake blocked threads!) + * E.g. unlock with contention, or + * enqueue with dequeuing threads + * + * Event count *optimization*: + * + * We prevent the race above using an epoch (count) for the events + * thus the name. However, to avoid rapid wraparound on the epoch, we + * only increment the epoch when the race condition is possible. That + * is to say, we only increment the event count when the + * data-structure has blocked threads. This not only delays + * wraparound, it also will avoid an atomic instruction for all + * operations that don't involve blocked threads (a common-case, + * exemplified by futexes, for example). + * + * Usage optimization: + * + * Because of the event counter optimization to only use expensive + * operations when triggering there are blocked threads, the user of + * this API can trigger whenever transitioning back to S0. + */ + +struct crt_blkpt { + sched_blkpt_id_t id; + /* most significant bit specifies blocked thds */ + sched_blkpt_epoch_t epoch_blocked; +}; + +struct crt_blkpt_checkpoint { + sched_blkpt_epoch_t epoch_blocked; +}; + +typedef enum +{ + CRT_BLKPT_UNIPROC = 1, /* are the event operations only called on a single core? */ + CRT_BLKPT_CRIT_SECT = 2, /* is only one thread ever going to trigger at a time? */ + CRT_BLKPT_WAKE_ALL = 4 +} crt_blkpt_flags_t; + +#define CRT_BLKPT_EPOCH_BLKED_BITS (sizeof(sched_blkpt_epoch_t) * 8) +#define CRT_BLKPT_BLKED_MASK (1 << (CRT_BLKPT_EPOCH_BLKED_BITS - 2)) +#define CRT_BLKPT_BLKED(e) ((e)&CRT_BLKPT_BLKED_MASK) +#define CRT_BLKPT_EPOCH(e) ((e) & ~CRT_BLKPT_BLKED_MASK) + +/* Return != 0 on failure: no ids to allocate */ +static inline int +crt_blkpt_init(struct crt_blkpt *blkpt) +{ + sched_blkpt_id_t id; + + id = sched_blkpt_alloc(); + if (id == SCHED_BLKPT_NULL) return -1; + + *blkpt = (struct crt_blkpt){.id = id, .epoch_blocked = 0}; + + return 0; +} + +static inline int +crt_blkpt_teardown(struct crt_blkpt *blkpt) +{ + return sched_blkpt_free(blkpt->id); +} + +// returns IF updated chkpt and unmasks block value +/* Internal APIs that must be inlined to remove the branches */ +static inline int +__crt_blkpt_atomic_trigger(sched_blkpt_epoch_t *ec, sched_blkpt_epoch_t chkpt, crt_blkpt_flags_t flags) +{ + /* + * Assume that the most significant bit is the blocked + * indicator. This math might reset it to zero, which we want + * to do anyway (as part of CRT_BLKPT_EPOCH). + */ + sched_blkpt_epoch_t new = CRT_BLKPT_EPOCH(chkpt + 1); + + /* inlined so that constant propagation will get rid of condition */ + if (flags == CRT_BLKPT_UNIPROC) { + return ps_upcas(ec, chkpt, new); + } else { + return ps_cas(ec, chkpt, new); + } + /* TODO: faa for CRT_BLKPT_CRIT_SECT? */ +} + +/* + * If we return 1, then the caller will attempt to block, otherwise, + * return 0 and it will re-check the data-structure assuming that + * something happened in the mean time. + */ +static inline int +__crt_blkpt_atomic_wait(sched_blkpt_epoch_t *ec, sched_blkpt_epoch_t chkpt, crt_blkpt_flags_t flags) +{ + sched_blkpt_epoch_t cached = ps_load(ec); + sched_blkpt_epoch_t new = cached | CRT_BLKPT_BLKED_MASK; + int ret; + + /* + * We are the second or later blocker. Blocked already + * set. We're done here. + * + * It isn't clear if it is better to have the additional + * branch here for this to avoid atomic instructions, or to + * just always do the atomic instructions and possibly fail. + */ + if (cached == new) return 1; + + /* function is inlined so that constant propagation will get rid of condition */ + if (flags == CRT_BLKPT_UNIPROC) { + ret = ps_upcas(ec, chkpt, new); + } else { + ret = ps_cas(ec, chkpt, new); + } + if (unlikely(!ret)) { + /* + * CAS failure can mean that 1. another thread + * blocked, and set the blocked bit, or 2. an event is + * triggered. In the former case, we still want to + * block. In the latter case, we want to go back to + * the data-structure. + */ + return ps_load(ec) == new; /* same epoch with blocked set? == success */ + } + + return 1; +} + +/* Trigger an event, waking blocked threads. */ +static inline void +crt_blkpt_trigger(struct crt_blkpt *blkpt, crt_blkpt_flags_t flags) +{ + /* + * Note that the flags should likely be passed in statically, + * as constants. That way they will be inlined the conditions + * in the *_atomic_* function will be removed. + */ + int wake_single; + + sched_blkpt_epoch_t saved = ps_load(&blkpt->epoch_blocked); + + + /* + 1. do trigger and update epoch see if blocked threads + 2. if blocked threads proceed + 3. if no blocked threads return + 4, + */ + + + do { + saved = ps_load(&blkpt->epoch_blocked); + } while (!__crt_blkpt_atomic_trigger(&blkpt->epoch_blocked, saved, flags)); + + if (likely(!CRT_BLKPT_BLKED(saved))) return; + + wake_single = (flags & CRT_BLKPT_WAKE_ALL) == 0; + sched_blkpt_trigger(blkpt->id, CRT_BLKPT_EPOCH(saved + 1), wake_single); +} + +/* Wake only a single, specified thread (tracked manually in the data-structure) */ +/* void crt_blkpt_trigger_one(struct crt_blkpt *blkpt, crt_blkpt_flags_t flags, cos_thdid_t thdid); */ + +/* + * Checkpoint the state of the current event counter. This checkpoint + * is the one that is active during our operations on the + * data-structure. If we determine that we want to wait for an event + * (thus blocking), then the state of the checkpoint will be compared + * versus the state of the event counter to see if we're working off + * of outdated information. + */ +static inline void +crt_blkpt_checkpoint(struct crt_blkpt *blkpt, struct crt_blkpt_checkpoint *chkpt) +{ + chkpt->epoch_blocked = ps_load(&blkpt->epoch_blocked); +} + +/* Wait for an event. */ +static inline void +crt_blkpt_wait(struct crt_blkpt *blkpt, crt_blkpt_flags_t flags, struct crt_blkpt_checkpoint *chkpt) +{ + /* + * If blocked is already set, we can try and block + * directly. Otherwise, go through and try to atomically set + * it. If that fails, then either epoch or blocked has been + * updated, so return and try accessing the data-structure + * again. + */ + if (!CRT_BLKPT_BLKED(chkpt->epoch_blocked) + && !__crt_blkpt_atomic_wait(&blkpt->epoch_blocked, chkpt->epoch_blocked, flags)) + return; + + if (unlikely(sched_blkpt_block(blkpt->id, CRT_BLKPT_EPOCH(chkpt->epoch_blocked), 0))) { + BUG(); /* we are using a blkpt id that doesn't exist! */ + } +} + +/* + * Create an execution dependency on the specified thread for, + * e.g. priority inheritance. + */ +/* void crt_blkpt_wait_dep(struct crt_blkpt *blkpt, crt_blkpt_flags_t flags, struct crt_blkpt_checkpoint *chkpt, + * cos_thdid_t thdid); */ + +#endif /* CRT_BLKPT_H */ diff --git a/src/components/include/crt_chan.h b/src/components/include/crt_chan.h new file mode 100644 index 0000000000..39a06974f4 --- /dev/null +++ b/src/components/include/crt_chan.h @@ -0,0 +1,233 @@ +/* + * Copyright 2019, Gabriel Parmer, GWU, gparmer@gwu.edu. + * + * This uses a two clause BSD License. + */ + +#ifndef CRT_CHAN_H +#define CRT_CHAN_H + +/*** + * + */ + +#include +#include +#include + +struct crt_chan { + u32_t producer; + /* If the ring is empty, recving threads will block on this blkpt. */ + struct crt_blkpt empty; + char _padding1[CACHE_LINE * 2 - (sizeof(struct crt_blkpt) + sizeof(u32_t))]; + u32_t consumer; + /* If the ring is full, sending thread will block on this blkpt. */ + struct crt_blkpt full; + char _padding2[CACHE_LINE * 2 - (sizeof(struct crt_blkpt) + sizeof(u32_t))]; + /* + * @item_sz is a power of two and corresponds to the + * wraparound_mask. The number of data items that the channel + * can hold is item_sz - 1. @wraparound_mask = nslots-1 (were + * nslots is a power of two) + */ + u32_t item_sz, wraparound_mask; + u32_t nslots; + /* The memory for the channel. */ + char mem[0]; +}; + +/* produce a */ +#define CRT_CHAN_STATIC_ALLOC(name, type, nslots) \ +struct __crt_chan_envelope_##name { \ + struct crt_chan c; \ + char mem[nslots * sizeof(type)]; \ +} __##name; \ +struct crt_chan *name = &__##name.c + +#define CRT_CHAN_TYPE_PROTOTYPES(name, type, nslots) \ +static inline int \ +crt_chan_init_##name(struct crt_chan *c) \ +{ return crt_chan_init(c, sizeof(type), nslots); } \ +static inline void \ +crt_chan_teardown_##name(struct crt_chan *c) \ +{ crt_chan_teardown(c); } \ +static inline int \ +crt_chan_empty_##name(struct crt_chan *c) \ +{ return __crt_chan_empty(c, nslots - 1); } \ +static inline int \ +crt_chan_full_##name(struct crt_chan *c) \ +{ return __crt_chan_full(c, nslots - 1); } \ +static inline int \ +crt_chan_send_##name(struct crt_chan *c, void *item) \ +{ \ + assert(pow2(nslots)); \ + return __crt_chan_send(c, item, nslots - 1, sizeof(type)); \ +} \ +static inline int \ +crt_chan_recv_##name(struct crt_chan *c, void *item) \ +{ \ + assert(pow2(nslots)); \ + return __crt_chan_recv(c, item, nslots - 1, sizeof(type)); \ +} \ +static inline int \ +crt_chan_async_send_##name(struct crt_chan *c, void *item) \ +{ \ + assert(pow2(nslots)); \ + if (__crt_chan_produce(c, item, nslots - 1, sizeof(type))) return -EAGAIN; \ + return 0; \ +} \ +static inline int \ +crt_chan_async_recv_##name(struct crt_chan *c, void *item) \ +{ \ + assert(pow2(nslots)); \ + if (__crt_chan_consume(c, item, nslots - 1, sizeof(type))) return -EAGAIN; \ + return 0; \ +} + +#define CRT_CHANCHAN_PROTOTYPES(nslots) \ +CRT_CHAN_TYPE_PROTOTYPES(chan, struct chan *, nslots + +static inline unsigned int +__crt_chan_buff_idx(struct crt_chan *c, u32_t v, u32_t wraparound_mask) +{ return v & wraparound_mask; } + +static inline int +__crt_chan_full(struct crt_chan *c, u32_t wraparound_mask) +{ return c->consumer == __crt_chan_buff_idx(c, c->producer + 1, wraparound_mask); } + +static inline int +__crt_chan_empty(struct crt_chan *c, u32_t wraparound_mask) +{ return c->producer == c->consumer; } + +static inline int +__crt_chan_produce(struct crt_chan *c, void *d, u32_t wraparound_mask, u32_t sz) +{ + if (__crt_chan_full(c, wraparound_mask)) return 1; + memcpy(c->mem + (__crt_chan_buff_idx(c, c->producer, wraparound_mask) * sz), d, sz); + c->producer++; + + return 0; +} + +static inline int +__crt_chan_consume(struct crt_chan *c, void *d, u32_t wraparound_mask, u32_t sz) +{ + void *ret; + + if (__crt_chan_empty(c, wraparound_mask)) return 1; + memcpy(d, c->mem + (__crt_chan_buff_idx(c, c->consumer, wraparound_mask) * sz), sz); + c->consumer++; + + return 0; +} + +/** + * The next two functions pass all of the variables in via arguments, + * so that we can use them for constant propagation along with + * inlining to get rid of the general memcpy code. + */ +static inline int +__crt_chan_send(struct crt_chan *c, void *item, u32_t wraparound_mask, u32_t item_sz) +{ + while (1) { + struct crt_blkpt_checkpoint chkpt; + + crt_blkpt_checkpoint(&c->full, &chkpt); + if (!__crt_chan_produce(c, item, wraparound_mask, item_sz)) { + /* success! */ + crt_blkpt_trigger(&c->empty, 0); + break; + } + crt_blkpt_wait(&c->full, 0, &chkpt); + } + + return 0; +} + +static inline int +__crt_chan_recv(struct crt_chan *c, void *item, u32_t wraparound_mask, u32_t item_sz) +{ + while (1) { + struct crt_blkpt_checkpoint chkpt; + + crt_blkpt_checkpoint(&c->empty, &chkpt); + if (!__crt_chan_consume(c, item, wraparound_mask, item_sz)) { + /* success! */ + crt_blkpt_trigger(&c->full, 0); + break; + } + crt_blkpt_wait(&c->empty, 0, &chkpt); + } + + return 0; +} + + +/* + * We need to know how much to malloc? This function returns that + * requirement. It assumes (and checks) that @slots is a power of two. + */ +static inline int +crt_chan_mem_sz(int item_sz, int slots) +{ + assert(pow2(slots)); + + return sizeof(struct crt_chan) + item_sz * slots; +} + +/* How many slots can we fit into an allocation of a specific mem_sz */ +static inline int +crt_chan_nslots(int item_sz, int mem_sz) +{ + return leqpow2((mem_sz - sizeof(struct crt_chan)) / item_sz); +} + +static inline int +crt_chan_init(struct crt_chan *c, int item_sz, int slots) +{ + assert(pow2(slots)); + if (crt_blkpt_init(&c->empty)) return -1; + if (crt_blkpt_init(&c->full)) return -1; + c->nslots = slots; + c->item_sz = item_sz; + c->wraparound_mask = slots - 1; /* slots is a pow2 */ + + return 0; +} + +static inline void +crt_chan_teardown(struct crt_chan *c) +{ + crt_blkpt_teardown(&c->empty); + crt_blkpt_teardown(&c->full); +} + +/* User-facing send and receive APIs: */ + +static inline int +crt_chan_send(struct crt_chan *c, void *item) +{ + return __crt_chan_send(c, item, c->wraparound_mask, c->item_sz); +} + +static inline int +crt_chan_recv(struct crt_chan *c, void *item) +{ + return __crt_chan_recv(c, item, c->wraparound_mask, c->item_sz); +} + +static inline int +crt_chan_async_send(struct crt_chan *c, void *item) +{ + if (__crt_chan_produce(c, item, c->wraparound_mask, c->item_sz)) return -EAGAIN; + return 0; +} + +static inline int +crt_chan_async_recv(struct crt_chan *c, void *item) +{ + if (__crt_chan_consume(c, item, c->wraparound_mask, c->item_sz)) return -EAGAIN; + return 0; +} + +#endif /* CRT_CHAN_H */ diff --git a/src/components/include/crt_lock.h b/src/components/include/crt_lock.h new file mode 100644 index 0000000000..d46d45f9a2 --- /dev/null +++ b/src/components/include/crt_lock.h @@ -0,0 +1,102 @@ +#ifndef CRT_LOCK_H +#define CRT_LOCK_H + +/*** + * Simple blocking lock. Uses blockpoints to enable the blocking and + * waking of contending threads. This has little to no intelligence, + * for example, not expressing dependencies for PI. + */ + +#include +#include + +struct crt_lock { + unsigned long owner; + struct crt_blkpt blkpt; +}; + +struct crt_sem { + unsigned long count; + unsigned long max_threads; + struct crt_blkpt blkpt; +}; + +static inline void +crt_sem_up(struct crt_sem *s) +{ + struct crt_blkpt_checkpoint chkpt; + + long unsigned int thd_num_cur = ps_faa(&s->count, 1); + + while (1) { + crt_blkpt_checkpoint(&s->blkpt, &chkpt); + + if (s->count < s->max_threads) return; + crt_blkpt_wait (&s->blkpt, 0, &chkpt); + } +} + +static inline void +crt_sem_init(struct crt_sem *s, int size) +{ + s->max_threads = size; + s->count = 0; +} + +static inline void +crt_sem_down(struct crt_sem *s) +{ + struct crt_blkpt_checkpoint chkpt; + + long unsigned int thd_num_cur = ps_faa(&s->count, -1); + + crt_blkpt_trigger(&s->blkpt, 0); +} + +/* static inline void */ +/* crt_mutex_init(struct crt_mutex *m) */ +/* { */ + +/* } */ + + +static inline int +crt_lock_init(struct crt_lock *l) +{ + l->owner = 0; + + return crt_blkpt_init(&l->blkpt); +} + +static inline int +crt_lock_teardown(struct crt_lock *l) +{ + assert(l->owner == 0); + + return crt_blkpt_teardown(&l->blkpt); +} + +static inline void +crt_lock_take(struct crt_lock *l) +{ + struct crt_blkpt_checkpoint chkpt; + + while (1) { + crt_blkpt_checkpoint(&l->blkpt, &chkpt); + + if (ps_cas(&l->owner, 0, (unsigned long)cos_thdid())) { return; /* success! */ } + /* failure: try and block */ + crt_blkpt_wait(&l->blkpt, 0, &chkpt); + } +} + +static inline void +crt_lock_release(struct crt_lock *l) +{ + assert(l->owner == cos_thdid()); + l->owner = 0; + /* if there are blocked threads, wake 'em up! */ + crt_blkpt_trigger(&l->blkpt, 0); +} + +#endif /* CRT_LOCK_H */ diff --git a/src/components/include/sl.h b/src/components/include/sl.h index 1529c7835c..f8c21e2259 100644 --- a/src/components/include/sl.h +++ b/src/components/include/sl.h @@ -70,6 +70,10 @@ struct sl_global_cpu { extern struct sl_global_cpu sl_global_cpu_data[]; +typedef u32_t sched_blkpt_id_t; +#define SCHED_BLKPT_NULL 0 +typedef word_t sched_blkpt_epoch_t; + static inline struct sl_global_cpu * sl__globals_cpu(void) { @@ -120,6 +124,10 @@ sl_thdid(void) return tid; } +sched_blkpt_id_t sched_blkpt_alloc(void); +int sched_blkpt_free(sched_blkpt_id_t id); +int sched_blkpt_trigger(sched_blkpt_id_t blkpt, sched_blkpt_epoch_t epoch, int single); +int sched_blkpt_block(sched_blkpt_id_t blkpt, sched_blkpt_epoch_t epoch, thdid_t dependency); static inline struct sl_thd * sl_thd_curr(void) diff --git a/src/components/include/stacklist.h b/src/components/include/stacklist.h new file mode 100644 index 0000000000..8651a408a7 --- /dev/null +++ b/src/components/include/stacklist.h @@ -0,0 +1,94 @@ +#ifndef STACKLIST_H +#define STACKLIST_H + +/** + * Modified to support multi-core via a Treiber stack. This is not 100% + * a great solution as it isn't FIFO. However, we release *all* + * threads when unlocking, so the priority scheduling should take over + * at that point. + */ + +#include +#include + +struct stacklist { + thdid_t thdid; + struct stacklist *next; +}; + +struct stacklist_head { + struct stacklist *head; +}; + +static inline void +stacklist_init(struct stacklist_head *h) +{ + h->head = NULL; +} + +/* + * Remove a thread from the list that has been woken. Return 0 on + * success, and 1 if it could not be removed. + */ +static inline int +stacklist_rem(struct stacklist *l) +{ + /* + * Not currently supported with Trebor Stack. Threads that + * wake early still have to wait their turn. + */ + return 1; +} + +/* Add a thread that is going to block */ +static inline void +stacklist_add(struct stacklist_head *h, struct stacklist *l) +{ + l->thdid = cos_thdid(); + l->next = NULL; + assert(h); + + while (1) { + struct stacklist *n = ps_load(&h->head); + + l->next = n; + if (ps_cas((unsigned long *)&h->head, (unsigned long)n, (unsigned long)l)) break; + } +} + +/* Get a thread to wake up, and remove its record! */ +static inline thdid_t +stacklist_dequeue(struct stacklist_head *h) +{ + struct stacklist *sl; + + if (!h->head) return 0; + + /* + * Only a single thread should trigger an event, and dequeue + * threads, but we'll implement this conservatively. Given + * this, please note that this should *not* iterate more than + * once. + */ + while (1) { + sl = ps_load(&h->head); + + if (ps_cas((unsigned long *)&h->head, (unsigned long)sl, (unsigned long)sl->next)) break; + } + sl->next = NULL; + + return sl->thdid; +} + +/* + * A thread that wakes up after blocking using a stacklist should be + * able to assume that it is no longer on the list. This enables them + * to assert on that fact. + */ +static inline int +stacklist_is_removed(struct stacklist *l) +{ + return l->next == NULL; +} + +#endif /* STACKLIST_H */ diff --git a/src/components/lib/linux b/src/components/lib/linux new file mode 160000 index 0000000000..8a1fc6cf60 --- /dev/null +++ b/src/components/lib/linux @@ -0,0 +1 @@ +Subproject commit 8a1fc6cf60d853e9abf724a3ed27d5680fb5807f diff --git a/src/components/lib/sl/Makefile b/src/components/lib/sl/Makefile index 6e908cda0b..39859fd419 100644 --- a/src/components/lib/sl/Makefile +++ b/src/components/lib/sl/Makefile @@ -1,6 +1,6 @@ include Makefile.src Makefile.comp -LIB_OBJS=sl_capmgr.o sl_raw.o sl_sched.o sl_xcpu.o sl_child.o sl_mod_fprr.o sl_lock.o sl_thd_static_backend.o +LIB_OBJS=sl_capmgr.o sl_raw.o sl_sched.o sl_xcpu.o sl_child.o sl_mod_fprr.o sl_lock.o sl_thd_static_backend.o sl_blkpt.o LIBS=$(LIB_OBJS:%.o=%.a) CINC+=-m32 diff --git a/src/components/lib/sl/sl_blkpt.c b/src/components/lib/sl/sl_blkpt.c new file mode 100644 index 0000000000..4d09f0d9a4 --- /dev/null +++ b/src/components/lib/sl/sl_blkpt.c @@ -0,0 +1,134 @@ +#include +#include + +#define NBLKPTS 64 +struct blkpt_mem { + sched_blkpt_id_t id; + sched_blkpt_epoch_t epoch; + struct stacklist_head blocked; +}; +static struct blkpt_mem __blkpts[NBLKPTS]; +static int __blkpt_offset = 1; + +#define BLKPT_EPOCH_BLKED_BITS ((sizeof(sched_blkpt_epoch_t) * 8) +#define BLKPT_EPOCH_DIFF (BLKPT_EPOCH_BLKED_BITS - 2)/2) + +/* + * Is cmp > e? This is more complicated than it seems it should be + * only because of wrap-around. We have to consider the case that we + * have, and that we haven't wrapped around. + */ +static int +blkpt_epoch_is_higher(sched_blkpt_epoch_t e, sched_blkpt_epoch_t cmp) +{ + return (e > cmp && (e - cmp) > BLKPT_EPOCH_DIFF) || (e < cmp && (cmp - e) < BLKPT_EPOCH_DIFF); +} + +static struct blkpt_mem * +blkpt_get(sched_blkpt_id_t id) +{ + if (id - 1 == NBLKPTS) return NULL; + + return &__blkpts[id - 1]; +} + +sched_blkpt_id_t +sched_blkpt_alloc(void) +{ + sched_blkpt_id_t id; + struct blkpt_mem *m; + sched_blkpt_id_t ret = SCHED_BLKPT_NULL; + + sl_cs_enter(); + + id = (sched_blkpt_id_t)__blkpt_offset; + m = blkpt_get(id); + if (!m) ERR_THROW(SCHED_BLKPT_NULL, unlock); + + m->id = id; + ret = id; + m->epoch = 0; + stacklist_init(&m->blocked); + __blkpt_offset++; +unlock: + sl_cs_exit(); + + return ret; +} + +int +sched_blkpt_free(sched_blkpt_id_t id) +{ + /* alloc only for now */ + return 0; +} + +int +sched_blkpt_trigger(sched_blkpt_id_t blkpt, sched_blkpt_epoch_t epoch, int single) +{ + thdid_t tid; + struct sl_thd * t; + struct blkpt_mem *m; + int ret = 0; + + sl_cs_enter(); + + m = blkpt_get(blkpt); + if (!m) ERR_THROW(-1, unlock); + + /* is the new epoch more recent than the existing? */ + if (!blkpt_epoch_is_higher(m->epoch, epoch)) ERR_THROW(0, unlock); + + m->epoch = epoch; + while ((tid = stacklist_dequeue(&m->blocked)) != 0) { + t = sl_thd_lkup(tid); + assert(t); + + + + sl_thd_wakeup_no_cs(t); /* ignore retval: process next thread */ + + //we woke up one thread and return + if (single == 1) { return; } + } + /* most likely we switch to a woken thread here */ + sl_cs_exit_schedule(); + + return 0; +unlock: + sl_cs_exit(); + + return ret; +} + +int +sched_blkpt_block(sched_blkpt_id_t blkpt, sched_blkpt_epoch_t epoch, thdid_t dependency) +{ + struct blkpt_mem *m; + struct sl_thd * t; + struct stacklist sl; /* The stack-based structure we'll use to track ourself */ + int ret = 0; + + sl_cs_enter(); + + m = blkpt_get(blkpt); + if (!m) ERR_THROW(-1, unlock); + + /* Outdated event? don't block! */ + if (blkpt_epoch_is_higher(m->epoch, epoch)) ERR_THROW(0, unlock); + + /* Block! */ + stacklist_add(&m->blocked, &sl); + + t = sl_thd_curr(); + if (sl_thd_block_no_cs(t, SL_THD_BLOCKED, 0)) ERR_THROW(-1, unlock); + + sl_cs_exit_schedule(); + assert(stacklist_is_removed(&sl)); /* we cannot still be on the list */ + + return 0; +unlock: + sl_cs_exit(); + + return ret; +} diff --git a/src/platform/i386/runscripts/crttests.sh b/src/platform/i386/runscripts/crttests.sh new file mode 100644 index 0000000000..55c6b0792b --- /dev/null +++ b/src/platform/i386/runscripts/crttests.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cp tests.crt_tests.o llboot.o +./cos_linker "llboot.o, :" ./gen_client_stub