diff --git a/common.mk b/common.mk index a2ae9fe3aa..839354118a 100755 --- a/common.mk +++ b/common.mk @@ -917,8 +917,10 @@ strlcat.$(OBJEXT): {$(VPATH)}strlcat.c strlcpy.$(OBJEXT): {$(VPATH)}strlcpy.c strstr.$(OBJEXT): {$(VPATH)}strstr.c nt.$(OBJEXT): {$(VPATH)}nt.c + +COROUTINE_SRC = $(COROUTINE_H:.h=).$(ASMEXT) .coroutine_obj $(COROUTINE_OBJ): \ - {$(VPATH)}$(COROUTINE_H:.h=).$(ASMEXT) \ + {$(VPATH)}$(COROUTINE_SRC:/ucontext/Context.S=/ucontext/Context.c) \ $(COROUTINE_H:/Context.h=/.time) $(COROUTINE_H:/Context.h=/.time): $(Q) $(MAKEDIRS) $(@D) diff --git a/configure.ac b/configure.ac index 8c431ad70b..3c87f78d44 100644 --- a/configure.ac +++ b/configure.ac @@ -2263,7 +2263,7 @@ AS_CASE(["$rb_cv_fiber_coroutine"], [yes|''], [ AS_CASE(["$ac_cv_sizeof_voidp"], [8], [ rb_cv_fiber_coroutine=amd64 ], [4], [ rb_cv_fiber_coroutine=x86 ], - [*], [ rb_cv_fiber_coroutine= ] + [*], [ rb_cv_fiber_coroutine= ] ) ], [*86-linux], [ @@ -2282,7 +2282,7 @@ AS_CASE(["$rb_cv_fiber_coroutine"], [yes|''], [ rb_cv_fiber_coroutine=ppc64le ], [*], [ - rb_cv_fiber_coroutine= + rb_cv_fiber_coroutine=ucontext ] ) AC_MSG_RESULT(${rb_cv_fiber_coroutine:-no}) diff --git a/cont.c b/cont.c index f43de10eeb..4622f948ff 100644 --- a/cont.c +++ b/cont.c @@ -187,7 +187,7 @@ struct rb_fiber_struct { #if FIBER_USE_NATIVE #if defined(FIBER_USE_COROUTINE) #define FIBER_ALLOCATE_STACK - coroutine_context context; + struct coroutine_context context; void *ss_sp; size_t ss_size; #elif defined(_WIN32) @@ -802,7 +802,7 @@ cont_restore_thread(rb_context_t *cont) #if FIBER_USE_NATIVE #if defined(FIBER_USE_COROUTINE) static COROUTINE -fiber_entry(coroutine_context * from, coroutine_context * to) +fiber_entry(struct coroutine_context * from, struct coroutine_context * to) { rb_fiber_start(); } @@ -914,7 +914,7 @@ fiber_initialize_machine_stack_context(rb_fiber_t *fib, size_t size) ptr = fiber_machine_stack_alloc(size); fib->ss_sp = ptr; fib->ss_size = size; - coroutine_initialize(&fib->context, fiber_entry, ptr+size, size); + coroutine_initialize(&fib->context, fiber_entry, ptr, size); sec->machine.stack_start = (VALUE*)(ptr + STACK_DIR_UPPER(0, size)); sec->machine.stack_maxsize = size - RB_PAGE_SIZE; #elif defined(_WIN32) @@ -1530,7 +1530,7 @@ root_fiber_alloc(rb_thread_t *th) #if FIBER_USE_NATIVE #if defined(FIBER_USE_COROUTINE) - coroutine_initialize(&fib->context, NULL, NULL, 0); + coroutine_initialize_main(&fib->context); #elif defined(_WIN32) /* setup fib_handle for root Fiber */ if (fib->fib_handle == 0) { diff --git a/coroutine/amd64/Context.S b/coroutine/amd64/Context.S index 42fe79c164..ac986b2aa5 100644 --- a/coroutine/amd64/Context.S +++ b/coroutine/amd64/Context.S @@ -27,7 +27,7 @@ PREFIXED_SYMBOL(SYMBOL_PREFIX,coroutine_transfer): # Restore callee stack pointer movq (%rsi), %rsp - # Restore callee stack + # Restore callee state popq %r15 popq %r14 popq %r13 diff --git a/coroutine/amd64/Context.h b/coroutine/amd64/Context.h index 4ae31d157b..8fe323c1a1 100644 --- a/coroutine/amd64/Context.h +++ b/coroutine/amd64/Context.h @@ -10,35 +10,32 @@ #include #include -#if __cplusplus -extern "C" { -#endif - #define COROUTINE __attribute__((noreturn)) void enum {COROUTINE_REGISTERS = 6}; -typedef struct +struct coroutine_context { void **stack_pointer; -} coroutine_context; +}; -typedef COROUTINE(* coroutine_start)(coroutine_context *from, coroutine_context *self); +typedef COROUTINE(* coroutine_start)(struct coroutine_context *from, struct coroutine_context *self); + +static inline void coroutine_initialize_main(struct coroutine_context * context) { + context->stack_pointer = NULL; +} static inline void coroutine_initialize( - coroutine_context *context, + struct coroutine_context *context, coroutine_start start, - void *stack_pointer, - size_t stack_size + void *stack, + size_t size ) { - /* Force 16-byte alignment */ - context->stack_pointer = (void**)((uintptr_t)stack_pointer & ~0xF); + assert(start && stack && size >= 1024); - if (!start) { - assert(!context->stack_pointer); - /* We are main coroutine for this thread */ - return; - } + // Stack grows down. Force 16-byte alignment. + char * top = (char*)stack + size; + context->stack_pointer = (void**)((uintptr_t)top & ~0xF); *--context->stack_pointer = NULL; *--context->stack_pointer = (void*)start; @@ -47,13 +44,9 @@ static inline void coroutine_initialize( memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS); } -coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target); +struct coroutine_context * coroutine_transfer(struct coroutine_context * current, struct coroutine_context * target); -static inline void coroutine_destroy(coroutine_context * context) +static inline void coroutine_destroy(struct coroutine_context * context) { context->stack_pointer = NULL; } - -#if __cplusplus -} -#endif diff --git a/coroutine/arm32/Context.h b/coroutine/arm32/Context.h index 8105bd2cff..27f97908bc 100644 --- a/coroutine/arm32/Context.h +++ b/coroutine/arm32/Context.h @@ -10,34 +10,32 @@ #include #include -#if __cplusplus -extern "C" { -#endif - #define COROUTINE __attribute__((noreturn)) void enum {COROUTINE_REGISTERS = 8}; -typedef struct +struct coroutine_context { void **stack_pointer; -} coroutine_context; +}; -typedef COROUTINE(* coroutine_start)(coroutine_context *from, coroutine_context *self); +typedef COROUTINE(* coroutine_start)(struct coroutine_context *from, struct coroutine_context *self); + +static inline void coroutine_initialize_main(struct coroutine_context * context) { + context->stack_pointer = NULL; +} static inline void coroutine_initialize( - coroutine_context *context, + struct coroutine_context *context, coroutine_start start, - void *stack_pointer, - size_t stack_size + void *stack, + size_t size ) { - context->stack_pointer = (void**)stack_pointer; + assert(start && stack && size >= 1024); - if (!start) { - assert(!context->stack_pointer); - /* We are main coroutine for this thread */ - return; - } + // Stack grows down. Force 16-byte alignment. + char * top = (char*)stack + size; + context->stack_pointer = (void**)((uintptr_t)top & ~0xF); *--context->stack_pointer = (void*)start; @@ -45,12 +43,8 @@ static inline void coroutine_initialize( memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS); } -coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target); +struct coroutine_context * coroutine_transfer(struct coroutine_context * current, struct coroutine_context * target); -static inline void coroutine_destroy(coroutine_context * context) +static inline void coroutine_destroy(struct coroutine_context * context) { } - -#if __cplusplus -} -#endif diff --git a/coroutine/arm64/Context.h b/coroutine/arm64/Context.h index ed646e818b..a1ae921144 100644 --- a/coroutine/arm64/Context.h +++ b/coroutine/arm64/Context.h @@ -10,35 +10,32 @@ #include #include -#if __cplusplus -extern "C" { -#endif - #define COROUTINE __attribute__((noreturn)) void enum {COROUTINE_REGISTERS = 0xb0 / 8}; -typedef struct +struct coroutine_context { void **stack_pointer; -} coroutine_context; +}; -typedef COROUTINE(* coroutine_start)(coroutine_context *from, coroutine_context *self); +typedef COROUTINE(* coroutine_start)(struct coroutine_context *from, struct coroutine_context *self); + +static inline void coroutine_initialize_main(struct coroutine_context * context) { + context->stack_pointer = NULL; +} static inline void coroutine_initialize( - coroutine_context *context, + struct coroutine_context *context, coroutine_start start, - void *stack_pointer, - size_t stack_size + void *stack, + size_t size ) { - /* Force 16-byte alignment */ - context->stack_pointer = (void**)((uintptr_t)stack_pointer & ~0xF); + assert(start && stack && size >= 1024); - if (!start) { - assert(!context->stack_pointer); - /* We are main coroutine for this thread */ - return; - } + // Stack grows down. Force 16-byte alignment. + char * top = (char*)stack + size; + context->stack_pointer = (void**)((uintptr_t)top & ~0xF); context->stack_pointer -= COROUTINE_REGISTERS; memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS); @@ -46,12 +43,8 @@ static inline void coroutine_initialize( context->stack_pointer[0xa0 / 8] = (void*)start; } -coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target); +struct coroutine_context * coroutine_transfer(struct coroutine_context * current, struct coroutine_context * target); -static inline void coroutine_destroy(coroutine_context * context) +static inline void coroutine_destroy(struct coroutine_context * context) { } - -#if __cplusplus -} -#endif diff --git a/coroutine/ppc64le/Context.h b/coroutine/ppc64le/Context.h index 5971cd8a9b..adf21b4fd9 100644 --- a/coroutine/ppc64le/Context.h +++ b/coroutine/ppc64le/Context.h @@ -3,10 +3,6 @@ #include #include -#if __cplusplus -extern "C" { -#endif - #define COROUTINE __attribute__((noreturn)) void enum { @@ -15,27 +11,28 @@ enum { + 4 /* space for fiber_entry() to store the link register */ }; -typedef struct +struct coroutine_context { void **stack_pointer; -} coroutine_context; +}; -typedef COROUTINE(* coroutine_start)(coroutine_context *from, coroutine_context *self); +typedef COROUTINE(* coroutine_start)(struct coroutine_context *from, struct coroutine_context *self); + +static inline void coroutine_initialize_main(struct coroutine_context * context) { + context->stack_pointer = NULL; +} static inline void coroutine_initialize( - coroutine_context *context, + struct coroutine_context *context, coroutine_start start, - void *stack_pointer, - size_t stack_size + void *stack, + size_t size ) { - /* Force 16-byte alignment */ - context->stack_pointer = (void**)((uintptr_t)stack_pointer & ~0xF); + assert(start && stack && size >= 1024); - if (!start) { - assert(!context->stack_pointer); - /* We are main coroutine for this thread */ - return; - } + // Stack grows down. Force 16-byte alignment. + char * top = (char*)stack + size; + context->stack_pointer = (void**)((uintptr_t)top & ~0xF); context->stack_pointer -= COROUTINE_REGISTERS; memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS); @@ -44,13 +41,9 @@ static inline void coroutine_initialize( context->stack_pointer[18] = ((char*)start) + 8; } -coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target); +struct coroutine_context * coroutine_transfer(struct coroutine_context * current, struct coroutine_context * target); -static inline void coroutine_destroy(coroutine_context * context) +static inline void coroutine_destroy(struct coroutine_context * context) { context->stack_pointer = NULL; } - -#if __cplusplus -} -#endif diff --git a/coroutine/ucontext/Context.c b/coroutine/ucontext/Context.c new file mode 100644 index 0000000000..ab8fda923e --- /dev/null +++ b/coroutine/ucontext/Context.c @@ -0,0 +1,16 @@ +/* + * This file is part of the "Coroutine" project and released under the MIT License. + * + * Created by Samuel Williams on 24/6/2019. + * Copyright, 2019, by Samuel Williams. All rights reserved. +*/ + +#include "Context.h" + +void coroutine_trampoline(void * _start, void * _context) +{ + coroutine_start start = _start; + struct coroutine_context * context = _context; + + start(context->from, context); +} diff --git a/coroutine/ucontext/Context.h b/coroutine/ucontext/Context.h new file mode 100644 index 0000000000..bde9be302a --- /dev/null +++ b/coroutine/ucontext/Context.h @@ -0,0 +1,66 @@ +/* + * This file is part of the "Coroutine" project and released under the MIT License. + * + * Created by Samuel Williams on 24/6/2019. + * Copyright, 2019, by Samuel Williams. All rights reserved. +*/ + +#pragma once + +#include +#include +#include + +#define COROUTINE __attribute__((noreturn)) void + +struct coroutine_context +{ + ucontext_t state; + struct coroutine_context * from; +}; + +typedef COROUTINE(* coroutine_start)(struct coroutine_context *from, struct coroutine_context *self); + +COROUTINE coroutine_trampoline(void * _start, void * _context); + +static inline void coroutine_initialize_main(struct coroutine_context * context) { + context->from = NULL; + getcontext(&context->state); +} + +static inline void coroutine_initialize( + struct coroutine_context *context, + coroutine_start start, + void *stack, + size_t size +) { + assert(start && stack && size >= 1024); + + coroutine_initialize_main(context); + + context->state.uc_stack.ss_size = size; + // Despite what it's called, this is not actually a stack pointer. It points to the address of the stack allocation (the lowest address). + context->state.uc_stack.ss_sp = (char*)stack; + context->state.uc_stack.ss_flags = 0; + context->state.uc_link = NULL; + + makecontext(&context->state, (void(*)(void))coroutine_trampoline, 2, (void*)start, (void*)context); +} + +static inline struct coroutine_context * coroutine_transfer(struct coroutine_context * current, struct coroutine_context * target) +{ + struct coroutine_context * previous = target->from; + + target->from = current; + swapcontext(¤t->state, &target->state); + target->from = previous; + + return target; +} + +static inline void coroutine_destroy(struct coroutine_context * context) +{ + context->state.uc_stack.ss_sp = NULL; + context->state.uc_stack.ss_size = 0; + context->from = NULL; +} diff --git a/coroutine/win32/Context.h b/coroutine/win32/Context.h index 779a9a8938..fe1a8a3648 100644 --- a/coroutine/win32/Context.h +++ b/coroutine/win32/Context.h @@ -10,53 +10,47 @@ #include #include -#if __cplusplus -extern "C" { -#endif - #define COROUTINE __declspec(noreturn) void __fastcall /* This doesn't include thread information block */ enum {COROUTINE_REGISTERS = 4}; -typedef struct +struct coroutine_context { void **stack_pointer; -} coroutine_context; +}; -typedef void(__fastcall * coroutine_start)(coroutine_context *from, coroutine_context *self); +typedef void(__fastcall * coroutine_start)(struct coroutine_context *from, struct coroutine_context *self); + +static inline void coroutine_initialize_main(struct coroutine_context * context) { + context->stack_pointer = NULL; +} static inline void coroutine_initialize( - coroutine_context *context, + struct coroutine_context *context, coroutine_start start, - void *stack_pointer, - size_t stack_size + void *stack, + size_t size ) { - context->stack_pointer = (void**)stack_pointer; + assert(start && stack && size >= 1024); - if (!start) { - assert(!context->stack_pointer); - /* We are main coroutine for this thread */ - return; - } + // Stack grows down. Force 16-byte alignment. + char * top = (char*)stack + size; + context->stack_pointer = (void**)((uintptr_t)top & ~0xF); *--context->stack_pointer = (void*)start; /* Windows Thread Information Block */ *--context->stack_pointer = 0; /* fs:[0] */ - *--context->stack_pointer = (void*)stack_pointer; /* fs:[4] */ - *--context->stack_pointer = (void*)((char *)stack_pointer - stack_size); /* fs:[8] */ + *--context->stack_pointer = (void*)top; /* fs:[4] */ + *--context->stack_pointer = (void*)stack; /* fs:[8] */ context->stack_pointer -= COROUTINE_REGISTERS; memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS); } -coroutine_context * __fastcall coroutine_transfer(coroutine_context * current, coroutine_context * target); +struct coroutine_context * __fastcall coroutine_transfer(struct coroutine_context * current, struct coroutine_context * target); -static inline void coroutine_destroy(coroutine_context * context) +static inline void coroutine_destroy(struct coroutine_context * context) { } - -#if __cplusplus -} -#endif diff --git a/coroutine/win64/Context.h b/coroutine/win64/Context.h index eb4540b779..6bf2dc5b35 100644 --- a/coroutine/win64/Context.h +++ b/coroutine/win64/Context.h @@ -10,10 +10,6 @@ #include #include -#if __cplusplus -extern "C" { -#endif - #define COROUTINE __declspec(noreturn) void enum { @@ -21,29 +17,30 @@ enum { COROUTINE_XMM_REGISTERS = 1+10*2, }; -typedef struct +struct coroutine_context { void **stack_pointer; -} coroutine_context; +}; -typedef void(* coroutine_start)(coroutine_context *from, coroutine_context *self); +typedef void(* coroutine_start)(struct coroutine_context *from, struct coroutine_context *self); void coroutine_trampoline(); -static inline void coroutine_initialize( - coroutine_context *context, - coroutine_start start, - void *stack_pointer, - size_t stack_size -) { - /* Force 16-byte alignment */ - context->stack_pointer = (void**)((uintptr_t)stack_pointer & ~0xF); +static inline void coroutine_initialize_main(struct coroutine_context * context) { + context->stack_pointer = NULL; +} - if (!start) { - assert(!context->stack_pointer); - /* We are main coroutine for this thread */ - return; - } +static inline void coroutine_initialize( + struct coroutine_context *context, + coroutine_start start, + void *stack, + size_t size +) { + assert(start && stack && size >= 1024); + + // Stack grows down. Force 16-byte alignment. + char * top = (char*)stack + size; + context->stack_pointer = (void**)((uintptr_t)top & ~0xF); /* Win64 ABI requires space for arguments */ context->stack_pointer -= 4; @@ -55,20 +52,16 @@ static inline void coroutine_initialize( /* Windows Thread Information Block */ /* *--context->stack_pointer = 0; */ /* gs:[0x00] is not used */ - *--context->stack_pointer = (void*)stack_pointer; /* gs:[0x08] */ - *--context->stack_pointer = (void*)((char *)stack_pointer - stack_size); /* gs:[0x10] */ + *--context->stack_pointer = (void*)top; /* gs:[0x08] */ + *--context->stack_pointer = (void*)stack; /* gs:[0x10] */ context->stack_pointer -= COROUTINE_REGISTERS; memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS); memset(context->stack_pointer - COROUTINE_XMM_REGISTERS, 0, sizeof(void*) * COROUTINE_XMM_REGISTERS); } -coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target); +struct coroutine_context * coroutine_transfer(struct coroutine_context * current, struct coroutine_context * target); -static inline void coroutine_destroy(coroutine_context * context) +static inline void coroutine_destroy(struct coroutine_context * context) { } - -#if __cplusplus -} -#endif diff --git a/coroutine/x86/Context.h b/coroutine/x86/Context.h index b471ecd9fe..a783dd296f 100644 --- a/coroutine/x86/Context.h +++ b/coroutine/x86/Context.h @@ -10,35 +10,32 @@ #include #include -#if __cplusplus -extern "C" { -#endif - #define COROUTINE __attribute__((noreturn, fastcall)) void enum {COROUTINE_REGISTERS = 4}; -typedef struct +struct coroutine_context { void **stack_pointer; -} coroutine_context; +}; -typedef COROUTINE(* coroutine_start)(coroutine_context *from, coroutine_context *self) __attribute__((fastcall)); +typedef COROUTINE(* coroutine_start)(struct coroutine_context *from, struct coroutine_context *self) __attribute__((fastcall)); + +static inline void coroutine_initialize_main(struct coroutine_context * context) { + context->stack_pointer = NULL; +} static inline void coroutine_initialize( - coroutine_context *context, + struct coroutine_context *context, coroutine_start start, - void *stack_pointer, - size_t stack_size + void *stack, + size_t size ) { - /* Force 16-byte alignment */ - context->stack_pointer = (void**)((uintptr_t)stack_pointer & ~0xF); + assert(start && stack && size >= 1024); - if (!start) { - assert(!context->stack_pointer); - /* We are main coroutine for this thread */ - return; - } + // Stack grows down. Force 16-byte alignment. + char * top = (char*)stack + size; + context->stack_pointer = (void**)((uintptr_t)top & ~0xF); *--context->stack_pointer = NULL; *--context->stack_pointer = (void*)start; @@ -47,13 +44,9 @@ static inline void coroutine_initialize( memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS); } -coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target) __attribute__((fastcall)); +struct coroutine_context * coroutine_transfer(struct coroutine_context * current, struct coroutine_context * target) __attribute__((fastcall)); -static inline void coroutine_destroy(coroutine_context * context) +static inline void coroutine_destroy(struct coroutine_context * context) { context->stack_pointer = NULL; } - -#if __cplusplus -} -#endif