Add global rust_get_current_task

Previously two methods existed: rust_sched_loop::get_task and rust_task::get_task_from_tcb. Merge both of them into one, trying the faster one (tcb) first, and if that fails, the slower one from the tls.

src/rt/rust_builtin.cpp

@@ -22,7 +22,7 @@ extern char **environ;
 
 extern "C" CDECL rust_str*
 last_os_error() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
 
     LOG(task, task, "last_os_error()");
 
@@ -65,7 +65,7 @@ last_os_error() {
 
 extern "C" CDECL rust_str *
 rust_getcwd() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, task, "rust_getcwd()");
 
     char cbuf[BUF_BYTES];
@@ -85,7 +85,7 @@ rust_getcwd() {
 #if defined(__WIN32__)
 extern "C" CDECL rust_vec *
 rust_env_pairs() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     size_t envc = 0;
     LPTCH ch = GetEnvironmentStringsA();
     LPTCH c;
@@ -111,7 +111,7 @@ rust_env_pairs() {
 #else
 extern "C" CDECL rust_vec *
 rust_env_pairs() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
 #ifdef __APPLE__
     char **environ = *_NSGetEnviron();
 #endif
@@ -133,21 +133,21 @@ refcount(intptr_t *v) {
 
 extern "C" CDECL void
 unsupervise() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     task->unsupervise();
 }
 
 extern "C" CDECL void
 vec_reserve_shared(type_desc* ty, rust_vec** vp,
                    size_t n_elts) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     reserve_vec_exact(task, vp, n_elts * ty->size);
 }
 
 extern "C" CDECL void
 str_reserve_shared(rust_vec** sp,
                    size_t n_elts) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     reserve_vec_exact(task, sp, n_elts + 1);
 }
 
@@ -157,7 +157,7 @@ str_reserve_shared(rust_vec** sp,
  */
 extern "C" CDECL rust_vec*
 vec_from_buf_shared(type_desc *ty, void *ptr, size_t count) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     size_t fill = ty->size * count;
     rust_vec* v = (rust_vec*)task->kernel->malloc(fill + sizeof(rust_vec),
                                                     "vec_from_buf");
@@ -168,7 +168,7 @@ vec_from_buf_shared(type_desc *ty, void *ptr, size_t count) {
 
 extern "C" CDECL void
 rust_str_push(rust_vec** sp, uint8_t byte) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     size_t fill = (*sp)->fill;
     reserve_vec(task, sp, fill + 1);
     (*sp)->data[fill-1] = byte;
@@ -178,7 +178,7 @@ rust_str_push(rust_vec** sp, uint8_t byte) {
 
 extern "C" CDECL void *
 rand_new() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     rust_sched_loop *thread = task->sched_loop;
     randctx *rctx = (randctx *) task->malloc(sizeof(randctx), "randctx");
     if (!rctx) {
@@ -196,7 +196,7 @@ rand_next(randctx *rctx) {
 
 extern "C" CDECL void
 rand_free(randctx *rctx) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     task->free(rctx);
 }
 
@@ -242,7 +242,7 @@ debug_abi_2(floats f) {
 static void
 debug_tydesc_helper(type_desc *t)
 {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, stdlib, "  size %" PRIdPTR ", align %" PRIdPTR
         ", first_param 0x%" PRIxPTR,
         t->size, t->align, t->first_param);
@@ -250,14 +250,14 @@ debug_tydesc_helper(type_desc *t)
 
 extern "C" CDECL void
 debug_tydesc(type_desc *t) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, stdlib, "debug_tydesc");
     debug_tydesc_helper(t);
 }
 
 extern "C" CDECL void
 debug_opaque(type_desc *t, uint8_t *front) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, stdlib, "debug_opaque");
     debug_tydesc_helper(t);
     // FIXME may want to actually account for alignment.  `front` may not
@@ -277,7 +277,7 @@ struct rust_box {
 
 extern "C" CDECL void
 debug_box(type_desc *t, rust_box *box) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, stdlib, "debug_box(0x%" PRIxPTR ")", box);
     debug_tydesc_helper(t);
     LOG(task, stdlib, "  refcount %" PRIdPTR,
@@ -294,7 +294,7 @@ struct rust_tag {
 
 extern "C" CDECL void
 debug_tag(type_desc *t, rust_tag *tag) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
 
     LOG(task, stdlib, "debug_tag");
     debug_tydesc_helper(t);
@@ -312,7 +312,7 @@ struct rust_fn {
 
 extern "C" CDECL void
 debug_fn(type_desc *t, rust_fn *fn) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, stdlib, "debug_fn");
     debug_tydesc_helper(t);
     LOG(task, stdlib, "  thunk at 0x%" PRIxPTR, fn->thunk);
@@ -326,7 +326,7 @@ extern "C" CDECL void *
 debug_ptrcast(type_desc *from_ty,
               type_desc *to_ty,
               void *ptr) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, stdlib, "debug_ptrcast from");
     debug_tydesc_helper(from_ty);
     LOG(task, stdlib, "to");
@@ -336,13 +336,13 @@ debug_ptrcast(type_desc *from_ty,
 
 extern "C" CDECL void *
 debug_get_stk_seg() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     return task->stk;
 }
 
 extern "C" CDECL rust_vec*
 rust_list_files(rust_str *path) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     array_list<rust_str*> strings;
 #if defined(__WIN32__)
     WIN32_FIND_DATA FindFileData;
@@ -443,20 +443,20 @@ precise_time_ns(uint64_t *ns) {
 
 extern "C" CDECL rust_sched_id
 rust_get_sched_id() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     return task->sched->get_id();
 }
 
 extern "C" CDECL rust_sched_id
 rust_new_sched(uintptr_t threads) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     assert(threads > 0 && "Can't create a scheduler with no threads, silly!");
     return task->kernel->create_scheduler(threads);
 }
 
 extern "C" CDECL rust_task_id
 get_task_id() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     return task->id;
 }
 
@@ -467,13 +467,13 @@ new_task_common(rust_scheduler *sched, rust_task *parent) {
 
 extern "C" CDECL rust_task*
 new_task() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     return new_task_common(task->sched, task);
 }
 
 extern "C" CDECL rust_task*
 rust_new_task_in_sched(rust_sched_id id) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     rust_scheduler *sched = task->kernel->get_scheduler_by_id(id);
     // FIXME: What if we didn't get the scheduler?
     return new_task_common(sched, task);
@@ -486,7 +486,7 @@ rust_task_config_notify(rust_task *target, rust_port_id *port) {
 
 extern "C" rust_task *
 rust_get_task() {
-    return rust_sched_loop::get_task();
+    return rust_get_current_task();
 }
 
 extern "C" CDECL void
@@ -496,13 +496,13 @@ start_task(rust_task *target, fn_env_pair *f) {
 
 extern "C" CDECL int
 sched_threads() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     return task->sched->number_of_threads();
 }
 
 extern "C" CDECL rust_port*
 new_port(size_t unit_sz) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, comm, "new_port(task=0x%" PRIxPTR " (%s), unit_sz=%d)",
         (uintptr_t) task, task->name, unit_sz);
     // port starts with refcount == 1
@@ -511,7 +511,7 @@ new_port(size_t unit_sz) {
 
 extern "C" CDECL void
 rust_port_begin_detach(rust_port *port, uintptr_t *yield) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, comm, "rust_port_detach(0x%" PRIxPTR ")", (uintptr_t) port);
     port->begin_detach(yield);
 }
@@ -523,7 +523,7 @@ rust_port_end_detach(rust_port *port) {
 
 extern "C" CDECL void
 del_port(rust_port *port) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG(task, comm, "del_port(0x%" PRIxPTR ")", (uintptr_t) port);
     delete port;
 }
@@ -541,7 +541,7 @@ get_port_id(rust_port *port) {
 extern "C" CDECL uintptr_t
 rust_port_id_send(type_desc *t, rust_port_id target_port_id, void *sptr) {
     bool sent = false;
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
 
     LOG(task, comm, "rust_port_id*_send port: 0x%" PRIxPTR,
         (uintptr_t) target_port_id);
@@ -572,14 +572,14 @@ port_recv(uintptr_t *dptr, rust_port *port, uintptr_t *yield) {
 extern "C" CDECL void
 rust_port_select(rust_port **dptr, rust_port **ports,
                  size_t n_ports, uintptr_t *yield) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     rust_port_selector *selector = task->get_port_selector();
     selector->select(task, dptr, ports, n_ports, yield);
 }
 
 extern "C" CDECL void
 rust_set_exit_status(intptr_t code) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     task->kernel->set_exit_status((int)code);
 }
 
@@ -594,7 +594,7 @@ extern void log_console_off(rust_env *env);
 
 extern "C" CDECL void
 rust_log_console_off() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     log_console_off(task->kernel->env);
 }
 

src/rt/rust_sched_loop.h

@@ -38,13 +38,6 @@ private:
 
     const int id;
 
-#ifndef __WIN32__
-    static pthread_key_t task_key;
-#else
-    static DWORD task_key;
-#endif
-
-    static bool tls_initialized;
     context c_context;
 
     bool should_exit;
@@ -69,6 +62,13 @@ private:
 public:
     rust_kernel *kernel;
     rust_scheduler *sched;
+    static bool tls_initialized;
+
+#ifndef __WIN32__
+    static pthread_key_t task_key;
+#else
+    static DWORD task_key;
+#endif
 
     // NB: this is used to filter *runtime-originating* debug
     // logging, on a per-scheduler basis. It's not likely what
@@ -116,8 +116,6 @@ public:
     void init_tls();
     void place_task_in_tls(rust_task *task);
 
-    static rust_task *get_task();
-
     // Called by each task when they are ready to be destroyed
     void release_task(rust_task *task);
 
@@ -134,33 +132,6 @@ rust_sched_loop::get_log() {
     return _log;
 }
 
-// This stuff is on the stack-switching fast path
-
-#ifndef __WIN32__
-
-inline rust_task *
-rust_sched_loop::get_task() {
-    if (!tls_initialized)
-        return NULL;
-    rust_task *task = reinterpret_cast<rust_task *>
-        (pthread_getspecific(task_key));
-    assert(task && "Couldn't get the task from TLS!");
-    return task;
-}
-
-#else
-
-inline rust_task *
-rust_sched_loop::get_task() {
-    if (!tls_initialized)
-        return NULL;
-    rust_task *task = reinterpret_cast<rust_task *>(TlsGetValue(task_key));
-    assert(task && "Couldn't get the task from TLS!");
-    return task;
-}
-
-#endif
-
 // NB: Runs on the Rust stack
 inline stk_seg *
 rust_sched_loop::borrow_c_stack() {

src/rt/rust_shape.cpp

@@ -552,7 +552,7 @@ extern "C" void
 shape_cmp_type(int8_t *result, const type_desc *tydesc,
                const type_desc **subtydescs, uint8_t *data_0,
                uint8_t *data_1, uint8_t cmp_type) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     shape::arena arena;
 
     // FIXME: This may well be broken when comparing two closures or objects
@@ -573,7 +573,7 @@ shape_cmp_type(int8_t *result, const type_desc *tydesc,
 
 extern "C" rust_str *
 shape_log_str(const type_desc *tydesc, uint8_t *data) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
 
     shape::arena arena;
     shape::type_param *params =
@@ -591,7 +591,7 @@ shape_log_str(const type_desc *tydesc, uint8_t *data) {
 
 extern "C" void
 shape_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
 
     shape::arena arena;
     shape::type_param *params =

src/rt/rust_task.h

@@ -270,7 +270,6 @@ public:
     const char *get_cond_name() { return cond_name; }
 
     void cleanup_after_turn();
-    static rust_task *get_task_from_tcb();
 };
 
 // This stuff is on the stack-switching fast path
@@ -445,26 +444,43 @@ rust_task::record_stack_limit() {
     record_sp_limit(stk->data + LIMIT_OFFSET + RED_ZONE_SIZE);
 }
 
-// The stack pointer boundary is stored in a quickly-accessible location
+inline rust_task* __rust_get_task_tls()
-// in the TCB. From that we can calculate the address of the stack segment
+{
-// structure it belongs to, and in that structure is a pointer to the task
+    if (!rust_sched_loop::tls_initialized)
-// that owns it.
+        return NULL;
-inline rust_task*
+#ifdef __WIN32__
-rust_task::get_task_from_tcb() {
+    rust_task *task = reinterpret_cast<rust_task *>
+        (TlsGetValue(rust_sched_loop::task_key));
+#else
+    rust_task *task = reinterpret_cast<rust_task *>
+        (pthread_getspecific(rust_sched_loop::task_key));
+#endif
+    assert(task && "Couldn't get the task from TLS!");
+    return task;
+
+}
+
+inline rust_task* rust_get_current_task() {
     uintptr_t sp_limit = get_sp_limit();
+
     // FIXME (1226) - Because of a hack in upcall_call_shim_on_c_stack this
     // value is sometimes inconveniently set to 0, so we can't use this
     // method of retreiving the task pointer and need to fall back to TLS.
-    if (sp_limit == 0) {
+    if (sp_limit == 0)
-        return NULL;
+        return __rust_get_task_tls();
-    }
 
+    // The stack pointer boundary is stored in a quickly-accessible location
+    // in the TCB. From that we can calculate the address of the stack segment
+    // structure it belongs to, and in that structure is a pointer to the task
+    // that owns it.
     uintptr_t seg_addr =
         sp_limit - RED_ZONE_SIZE - LIMIT_OFFSET - sizeof(stk_seg);
     stk_seg *stk = (stk_seg*) seg_addr;
     // Make sure we've calculated the right address
     ::check_stack_canary(stk);
-    assert(stk->task != NULL && "task pointer not in stack structure");
+
+    if (stk->task == NULL)
+        return __rust_get_task_tls();
     return stk->task;
 }
 

src/rt/rust_upcall.cpp

@@ -47,7 +47,7 @@ static void check_stack_alignment() { }
 inline void
 call_upcall_on_c_stack(void *args, void *fn_ptr) {
     check_stack_alignment();
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     task->call_on_c_stack(args, fn_ptr);
 }
 
@@ -62,7 +62,7 @@ extern "C" void record_sp_limit(void *limit);
  */
 extern "C" CDECL void
 upcall_call_shim_on_c_stack(void *args, void *fn_ptr) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
 
     // FIXME (1226) - The shim functions generated by rustc contain the
     // morestack prologue, so we need to let them know they have enough
@@ -85,7 +85,7 @@ upcall_call_shim_on_c_stack(void *args, void *fn_ptr) {
  */
 extern "C" CDECL void
 upcall_call_shim_on_rust_stack(void *args, void *fn_ptr) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
 
     // FIXME: Because of the hack in the other function that disables the
     // stack limit when entering the C stack, here we restore the stack limit
@@ -116,7 +116,7 @@ struct s_fail_args {
 
 extern "C" CDECL void
 upcall_s_fail(s_fail_args *args) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG_UPCALL_ENTRY(task);
     LOG_ERR(task, upcall, "upcall fail '%s', %s:%" PRIdPTR,
             args->expr, args->file, args->line);
@@ -142,7 +142,7 @@ struct s_malloc_args {
 
 extern "C" CDECL void
 upcall_s_malloc(s_malloc_args *args) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG_UPCALL_ENTRY(task);
 
     LOG(task, mem, "upcall malloc(0x%" PRIxPTR ")", args->td);
@@ -179,7 +179,7 @@ struct s_free_args {
 
 extern "C" CDECL void
 upcall_s_free(s_free_args *args) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG_UPCALL_ENTRY(task);
 
     rust_sched_loop *sched_loop = task->sched_loop;
@@ -225,7 +225,7 @@ struct s_shared_malloc_args {
 
 extern "C" CDECL void
 upcall_s_shared_malloc(s_shared_malloc_args *args) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG_UPCALL_ENTRY(task);
 
     LOG(task, mem, "upcall shared_malloc(%" PRIdPTR ")", args->nbytes);
@@ -253,7 +253,7 @@ struct s_shared_free_args {
 
 extern "C" CDECL void
 upcall_s_shared_free(s_shared_free_args *args) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG_UPCALL_ENTRY(task);
 
     rust_sched_loop *sched_loop = task->sched_loop;
@@ -277,7 +277,7 @@ struct s_shared_realloc_args {
 
 extern "C" CDECL void
 upcall_s_shared_realloc(s_shared_realloc_args *args) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG_UPCALL_ENTRY(task);
     args->retval = task->kernel->realloc(args->ptr, args->size);
 }
@@ -298,7 +298,7 @@ struct s_vec_grow_args {
 
 extern "C" CDECL void
 upcall_s_vec_grow(s_vec_grow_args *args) {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     LOG_UPCALL_ENTRY(task);
     reserve_vec(task, args->vp, args->new_sz);
     (*args->vp)->fill = args->new_sz;
@@ -320,7 +320,7 @@ extern "C" CDECL void
 upcall_s_str_concat(s_str_concat_args *args) {
     rust_vec *lhs = args->lhs;
     rust_vec *rhs = args->rhs;
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     size_t fill = lhs->fill + rhs->fill - 1;
     rust_vec* v = (rust_vec*)task->kernel->malloc(fill + sizeof(rust_vec),
                                                   "str_concat");
@@ -377,7 +377,7 @@ upcall_rust_personality(int version,
     s_rust_personality_args args = {(_Unwind_Reason_Code)0,
                                     version, actions, exception_class,
                                     ue_header, context};
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
 
     // The personality function is run on the stack of the
     // last function that threw or landed, which is going
@@ -444,7 +444,7 @@ upcall_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level) {
 // NB: This needs to be blazing fast. Don't switch stacks
 extern "C" CDECL void *
 upcall_new_stack(size_t stk_sz, void *args_addr, size_t args_sz) {
-    rust_task *task = rust_task::get_task_from_tcb();
+    rust_task *task = rust_get_current_task();
     return task->next_stack(stk_sz,
                             args_addr,
                             args_sz);
@@ -453,7 +453,7 @@ upcall_new_stack(size_t stk_sz, void *args_addr, size_t args_sz) {
 // NB: This needs to be blazing fast. Don't switch stacks
 extern "C" CDECL void
 upcall_del_stack() {
-    rust_task *task = rust_task::get_task_from_tcb();
+    rust_task *task = rust_get_current_task();
     task->prev_stack();
 }
 
@@ -463,7 +463,7 @@ upcall_del_stack() {
 // needs to acquire the value of the stack pointer
 extern "C" CDECL void
 upcall_reset_stack_limit() {
-    rust_task *task = rust_sched_loop::get_task();
+    rust_task *task = rust_get_current_task();
     task->reset_stack_limit();
 }
 

src/rt/rust_uv.cpp

@@ -20,13 +20,13 @@ struct handle_data {
 // helpers
 static void*
 current_kernel_malloc(size_t size, const char* tag) {
-  void* ptr = rust_sched_loop::get_task()->kernel->malloc(size, tag);
+  void* ptr = rust_get_current_task()->kernel->malloc(size, tag);
   return ptr;
 }
 
 static void
 current_kernel_free(void* ptr) {
-  rust_sched_loop::get_task()->kernel->free(ptr);
+  rust_get_current_task()->kernel->free(ptr);
 }
 
 static handle_data*