ZJIT: Write JITFrame pointer for C frames

cont.c

@@ -41,6 +41,7 @@ extern int madvise(caddr_t, size_t, int);
 #include "vm_sync.h"
 #include "id_table.h"
 #include "ractor_core.h"
+#include "zjit.h"
 
 static const int DEBUG = 0;
 
@@ -1347,8 +1348,11 @@ rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data)
 
         const rb_control_frame_t *cfp = cont->ec->cfp;
         while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) {
-            if (cfp->pc && cfp->iseq && imemo_type((VALUE)cfp->iseq) == imemo_iseq) {
-                callback(cfp->iseq, data);
+            if (rb_zjit_cfp_has_pc(cfp) && rb_zjit_cfp_has_iseq(cfp)) {
+                const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp);
+                if (iseq && imemo_type((VALUE)iseq) == imemo_iseq) {
+                    callback(iseq, data);
+                }
             }
             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
         }
@@ -1368,7 +1372,7 @@ rb_yjit_cancel_jit_return(void *leave_exit, void *leave_exception)
 
         const rb_control_frame_t *cfp = cont->ec->cfp;
         while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) {
-            if (cfp->jit_return && cfp->jit_return != leave_exception) {
+            if (CFP_JIT_RETURN(cfp) && cfp->jit_return != leave_exception) {
                 ((rb_control_frame_t *)cfp)->jit_return = leave_exit;
             }
             cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);

depend

@@ -2620,6 +2620,7 @@ cont.$(OBJEXT): {$(VPATH)}vm_debug.h
 cont.$(OBJEXT): {$(VPATH)}vm_opts.h
 cont.$(OBJEXT): {$(VPATH)}vm_sync.h
 cont.$(OBJEXT): {$(VPATH)}yjit.h
+cont.$(OBJEXT): {$(VPATH)}zjit.h
 debug.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h
 debug.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h
 debug.$(OBJEXT): $(CCAN_DIR)/list/list.h
@@ -5420,6 +5421,7 @@ error.$(OBJEXT): {$(VPATH)}vm_opts.h
 error.$(OBJEXT): {$(VPATH)}vm_sync.h
 error.$(OBJEXT): {$(VPATH)}warning.rbinc
 error.$(OBJEXT): {$(VPATH)}yjit.h
+error.$(OBJEXT): {$(VPATH)}zjit.h
 eval.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h
 eval.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h
 eval.$(OBJEXT): $(CCAN_DIR)/list/list.h
@@ -5689,6 +5691,7 @@ eval.$(OBJEXT): {$(VPATH)}vm_core.h
 eval.$(OBJEXT): {$(VPATH)}vm_debug.h
 eval.$(OBJEXT): {$(VPATH)}vm_opts.h
 eval.$(OBJEXT): {$(VPATH)}vm_sync.h
+eval.$(OBJEXT): {$(VPATH)}zjit.h
 explicit_bzero.$(OBJEXT): {$(VPATH)}config.h
 explicit_bzero.$(OBJEXT): {$(VPATH)}explicit_bzero.c
 explicit_bzero.$(OBJEXT): {$(VPATH)}internal/attr/format.h
@@ -8072,6 +8075,7 @@ jit.$(OBJEXT): {$(VPATH)}vm_core.h
 jit.$(OBJEXT): {$(VPATH)}vm_debug.h
 jit.$(OBJEXT): {$(VPATH)}vm_opts.h
 jit.$(OBJEXT): {$(VPATH)}vm_sync.h
+jit.$(OBJEXT): {$(VPATH)}zjit.h
 load.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h
 load.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h
 load.$(OBJEXT): $(CCAN_DIR)/list/list.h
@@ -18615,6 +18619,7 @@ thread.$(OBJEXT): {$(VPATH)}vm_core.h
 thread.$(OBJEXT): {$(VPATH)}vm_debug.h
 thread.$(OBJEXT): {$(VPATH)}vm_opts.h
 thread.$(OBJEXT): {$(VPATH)}vm_sync.h
+thread.$(OBJEXT): {$(VPATH)}zjit.h
 time.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h
 time.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h
 time.$(OBJEXT): $(CCAN_DIR)/list/list.h
@@ -20210,6 +20215,7 @@ vm_backtrace.$(OBJEXT): {$(VPATH)}vm_core.h
 vm_backtrace.$(OBJEXT): {$(VPATH)}vm_debug.h
 vm_backtrace.$(OBJEXT): {$(VPATH)}vm_opts.h
 vm_backtrace.$(OBJEXT): {$(VPATH)}vm_sync.h
+vm_backtrace.$(OBJEXT): {$(VPATH)}zjit.h
 vm_dump.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h
 vm_dump.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h
 vm_dump.$(OBJEXT): $(CCAN_DIR)/list/list.h
@@ -20458,6 +20464,7 @@ vm_dump.$(OBJEXT): {$(VPATH)}vm_core.h
 vm_dump.$(OBJEXT): {$(VPATH)}vm_debug.h
 vm_dump.$(OBJEXT): {$(VPATH)}vm_dump.c
 vm_dump.$(OBJEXT): {$(VPATH)}vm_opts.h
+vm_dump.$(OBJEXT): {$(VPATH)}zjit.h
 vm_sync.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h
 vm_sync.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h
 vm_sync.$(OBJEXT): $(CCAN_DIR)/list/list.h

error.c

@@ -50,6 +50,7 @@
 #include "ruby_assert.h"
 #include "vm_core.h"
 #include "yjit.h"
+#include "zjit.h"
 
 #include "builtin.h"
 
@@ -2367,7 +2368,7 @@ name_err_init_attr(VALUE exc, VALUE recv, VALUE method)
     rb_ivar_set(exc, id_name, method);
     err_init_recv(exc, recv);
     if (cfp && VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) {
-        rb_ivar_set(exc, id_iseq, rb_iseqw_new(cfp->iseq));
+        rb_ivar_set(exc, id_iseq, rb_iseqw_new(rb_zjit_cfp_iseq(cfp)));
     }
     return exc;
 }

eval.c

@@ -37,6 +37,7 @@
 #include "ruby/vm.h"
 #include "vm_core.h"
 #include "ractor_core.h"
+#include "zjit.h"
 
 NORETURN(static void rb_raise_jump(VALUE, VALUE));
 void rb_ec_clear_current_thread_trace_func(const rb_execution_context_t *ec);
@@ -2010,10 +2011,10 @@ errinfo_place(const rb_execution_context_t *ec)
 
     while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) {
         if (VM_FRAME_RUBYFRAME_P(cfp)) {
-            if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_RESCUE) {
+            if (ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type == ISEQ_TYPE_RESCUE) {
                 return &cfp->ep[VM_ENV_INDEX_LAST_LVAR];
             }
-            else if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_ENSURE &&
+            else if (ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type == ISEQ_TYPE_ENSURE &&
                      !THROW_DATA_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR]) &&
                      !FIXNUM_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR])) {
                 return &cfp->ep[VM_ENV_INDEX_LAST_LVAR];

gc.c

@@ -1000,11 +1000,11 @@ gc_validate_pc(VALUE obj)
 
     rb_execution_context_t *ec = GET_EC();
     const rb_control_frame_t *cfp = ec->cfp;
-    if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && cfp->pc) {
-        const VALUE *iseq_encoded = ISEQ_BODY(cfp->iseq)->iseq_encoded;
-        const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(cfp->iseq)->iseq_size;
-        RUBY_ASSERT(cfp->pc >= iseq_encoded, "PC not set when allocating, breaking tracing");
-        RUBY_ASSERT(cfp->pc <= iseq_encoded_end, "PC not set when allocating, breaking tracing");
+    if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && rb_zjit_cfp_has_pc(cfp)) {
+        const VALUE *iseq_encoded = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded;
+        const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_size;
+        RUBY_ASSERT(rb_zjit_cfp_pc(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing");
+        RUBY_ASSERT(rb_zjit_cfp_pc(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing");
     }
 #endif
 }

insns.def

@@ -926,7 +926,7 @@ opt_new
     // The bookkeeping slot should be empty.
     RUBY_ASSERT(TOPN(argc + 1) == Qnil);
 
-    if (vm_method_cfunc_is(GET_ISEQ(), cd, val, rb_class_new_instance_pass_kw)) {
+    if (vm_method_cfunc_is(GET_CFP(), cd, val, rb_class_new_instance_pass_kw)) {
         RB_DEBUG_COUNTER_INC(opt_new_hit);
         val = rb_obj_alloc(val);
         TOPN(argc) = val;
@@ -947,7 +947,7 @@ objtostring
 // attr bool leaf = false;
 // attr bool zjit_profile = true;
 {
-    val = vm_objtostring(GET_ISEQ(), recv, cd);
+    val = vm_objtostring(GET_CFP(), recv, cd);
 
     if (UNDEF_P(val)) {
         CALL_SIMPLE_METHOD();
@@ -1006,7 +1006,7 @@ opt_nil_p
 (VALUE val)
 // attr bool zjit_profile = true;
 {
-    val = vm_opt_nil_p(GET_ISEQ(), cd, recv);
+    val = vm_opt_nil_p(GET_CFP(), cd, recv);
 
     if (UNDEF_P(val)) {
         CALL_SIMPLE_METHOD();
@@ -1435,7 +1435,7 @@ opt_eq
 (VALUE val)
 // attr bool zjit_profile = true;
 {
-    val = opt_equality(GET_ISEQ(), recv, obj, cd);
+    val = opt_equality(GET_CFP(), recv, obj, cd);
 
     if (UNDEF_P(val)) {
         CALL_SIMPLE_METHOD();
@@ -1450,7 +1450,7 @@ opt_neq
 (VALUE val)
 // attr bool zjit_profile = true;
 {
-    val = vm_opt_neq(GET_ISEQ(), cd, cd_eq, recv, obj);
+    val = vm_opt_neq(GET_CFP(), cd, cd_eq, recv, obj);
 
     if (UNDEF_P(val)) {
         CALL_SIMPLE_METHOD();
@@ -1672,7 +1672,7 @@ opt_not
 (VALUE val)
 // attr bool zjit_profile = true;
 {
-    val = vm_opt_not(GET_ISEQ(), cd, recv);
+    val = vm_opt_not(GET_CFP(), cd, recv);
 
     if (UNDEF_P(val)) {
         CALL_SIMPLE_METHOD();

jit.c

@@ -19,6 +19,7 @@
 #include "internal/class.h"
 #include "internal/imemo.h"
 #include "ruby/internal/core/rtypeddata.h"
+#include "zjit.h"
 
 #ifndef _WIN32
 #include <sys/mman.h>
@@ -522,6 +523,10 @@ void
 rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc)
 {
     cfp->pc = pc;
+    if (rb_zjit_enabled_p/* && cfp->jit_return*/) {
+        //cfp->iseq = rb_zjit_jit_return_iseq(cfp->jit_return);
+        cfp->jit_return = 0; // TODO: do it in Rust (function_stub_hit)
+    }
 }
 
 void

proc.c

@@ -1285,6 +1285,9 @@ rb_proc_arity(VALUE self)
 static void
 block_setup(struct rb_block *block, VALUE block_handler)
 {
+    // TODO: do we need to do something about this for ZJIT?
+    // we copy captured block but not jit_return.
+    // do we need to query block_code for specialized C method calls?
     switch (vm_block_handler_type(block_handler)) {
       case block_handler_type_iseq:
         block->type = block_type_iseq;

thread.c

@@ -99,6 +99,7 @@
 #include "ractor_core.h"
 #include "vm_debug.h"
 #include "vm_sync.h"
+#include "zjit.h"
 
 #include "ccan/list/list.h"
 
@@ -5915,7 +5916,7 @@ update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg)
             VALUE num;
             void rb_iseq_clear_event_flags(const rb_iseq_t *iseq, size_t pos, rb_event_flag_t reset);
             if (GET_VM()->coverage_mode & COVERAGE_TARGET_ONESHOT_LINES) {
-                rb_iseq_clear_event_flags(cfp->iseq, cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE);
+                rb_iseq_clear_event_flags(cfp->iseq, rb_zjit_cfp_pc(cfp) - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE);
                 rb_ary_push(lines, LONG2FIX(line + 1));
                 return;
             }
@@ -5940,7 +5941,7 @@ update_branch_coverage(VALUE data, const rb_trace_arg_t *trace_arg)
     if (RB_TYPE_P(coverage, T_ARRAY) && !RBASIC_CLASS(coverage)) {
         VALUE branches = RARRAY_AREF(coverage, COVERAGE_INDEX_BRANCHES);
         if (branches) {
-            long pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1;
+            long pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1;
             long idx = FIX2INT(RARRAY_AREF(ISEQ_PC2BRANCHINDEX(cfp->iseq), pc)), count;
             VALUE counters = RARRAY_AREF(branches, 1);
             VALUE num = RARRAY_AREF(counters, idx);