Very WIP: Mutually recursive types

src/builtins.c

@@ -907,7 +907,9 @@ static int valid_type_param(jl_value_t *v)
     if (jl_is_vararg_type(v))
         return 0;
     // TODO: maybe more things
-    return jl_is_type(v) || jl_is_typevar(v) || jl_is_symbol(v) || jl_isbits(jl_typeof(v));
+    return jl_is_type(v) || jl_is_typevar(v) ||
+        jl_is_symbol(v) || jl_isbits(jl_typeof(v)) ||
+        jl_typeis(v, jl_placeholder_type);
 }
 
 JL_CALLABLE(jl_f_apply_type)
@@ -1308,6 +1310,7 @@ void jl_init_primitives(void) JL_GC_DISABLED
     add_builtin("TypeName", (jl_value_t*)jl_typename_type);
     add_builtin("DataType", (jl_value_t*)jl_datatype_type);
     add_builtin("TypeVar", (jl_value_t*)jl_tvar_type);
+    add_builtin("Placeholder", (jl_value_t*)jl_placeholder_type);
     add_builtin("UnionAll", (jl_value_t*)jl_unionall_type);
     add_builtin("Union", (jl_value_t*)jl_uniontype_type);
     add_builtin("TypeofBottom", (jl_value_t*)jl_typeofbottom_type);

src/datatype.c

@@ -84,6 +84,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(void)
     t->isdispatchtuple = 0;
     t->isbitstype = 0;
     t->zeroinit = 0;
+    t->incomplete = 0;
     t->isinlinealloc = 0;
     t->has_concrete_subtype = 1;
     t->layout = NULL;
@@ -503,6 +504,13 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(
     t->name->names = fnames;
     jl_gc_wb(t->name, t->name->names);
 
+    for (int i = 0; i < jl_svec_len(t->parameters); ++i) {
+        jl_value_t *p = jl_svecref(t->parameters, i);
+        if (jl_typeis(p, jl_placeholder_type)) {
+            dt_mark_incomplete(t, (jl_placeholder_t*)p);
+        }
+    }
+
     if (t->name->wrapper == NULL) {
         t->name->wrapper = (jl_value_t*)t;
         jl_gc_wb(t->name, t);

src/interpreter.c

@@ -101,7 +101,8 @@ SECT_INTERP static int equiv_type(jl_datatype_t *dta, jl_datatype_t *dtb)
 
 SECT_INTERP static void check_can_assign_type(jl_binding_t *b, jl_value_t *rhs)
 {
-    if (b->constp && b->value != NULL && jl_typeof(b->value) != jl_typeof(rhs))
+    if (b->constp && b->value != NULL && jl_typeof(b->value) != jl_typeof(rhs) &&
+        !jl_typeis(b->value, jl_placeholder_type))
         jl_errorf("invalid redefinition of constant %s",
                   jl_symbol_name(b->name));
 }
@@ -220,6 +221,178 @@ static void eval_primitivetype(jl_expr_t *ex, interpreter_state *s)
     JL_GC_POP();
 }
 
+/*
+ * We maintain a DAG of dependencies between incomplete datatypes. Each SCC in
+ * this graph is assigned a unique representative that represents the SCC to
+ * other SCCs. Obtain this representative for a given dt.
+ */
+static jl_datatype_t *get_scc_representative(jl_datatype_t *dt) {
+    if (dt->scc == dt)
+        return dt;
+    // For efficiency we only update the SCC representative when merging SCCs.
+    // This update here folds updating the members of the SCC into any
+    // subsequent access.
+    dt->scc = get_scc_representative(dt->scc);
+    return dt->scc;
+}
+
+ /*
+ * Check if adding a new dependency edge from scc_a to scc_b would add a cycle.
+ * If so, return the cycle.
+ *
+ * Note: We assume the SCC dag to be fairly small such that the search here is
+ * not too expensive. Should that not be the case in real world code, investigate
+ * using and incremental SCC maintenance algorithm (e.g.
+ * https://arxiv.org/pdf/1105.2397.pdf) instead.
+ */
+static jl_array_t *find_new_cycle(jl_datatype_t *scc_a, jl_datatype_t *scc_b) JL_NOTSAFEPOINT {
+    assert(get_scc_representative(scc_a) == scc_a);
+    assert(get_scc_representative(scc_b) == scc_b);
+    if (scc_a == scc_b) {
+        jl_array_t *cycle = jl_alloc_vec_any(0);
+        jl_array_ptr_1d_push(cycle, (jl_value_t*)scc_a);
+        return cycle;
+    }
+    for (int i = 0; i < jl_array_len(scc_a->depends); ++i) {
+        jl_array_t *cycle = find_new_cycle(
+            (jl_datatype_t*)jl_array_ptr_ref(scc_a->depends, i), scc_b);
+        if (cycle) {
+            jl_array_ptr_1d_push(cycle, (jl_value_t*)scc_a);
+            return cycle;
+        }
+    }
+    return NULL;
+}
+
+ static void filter_scc(jl_array_t *array, jl_datatype_t *scc_rep)
+{
+    size_t insert_point = 0;
+    for (int i = 0; i < jl_array_len(array); ++i) {
+        jl_datatype_t *connected_scc = (jl_datatype_t*)jl_array_ptr_ref(array, i);
+        if (get_scc_representative(connected_scc) == scc_rep) {
+            // If this SCC is now part of our new SCC, skip it in this list
+            continue;
+        }
+        if (insert_point != i) {
+            jl_array_ptr_set(array, insert_point,
+                jl_array_ptr_ref(array, i));
+        }
+        insert_point++;
+    }
+    jl_array_del_end(array, jl_array_len(array) - insert_point);
+}
+
+ static void filter_cycle_and_append(jl_array_t *newa, jl_array_t *olda, jl_datatype_t *scc_rep)
+{
+    for (int i = 0; i < jl_array_len(olda); ++i) {
+        jl_datatype_t *connected_scc = (jl_datatype_t*)jl_array_ptr_ref(olda, i);
+        if (get_scc_representative(connected_scc) != scc_rep)
+            jl_array_ptr_1d_push(newa, (jl_value_t*)connected_scc);
+    }
+}
+
+ static void incomplete_add_dep(jl_datatype_t *from, jl_datatype_t *to) {
+    assert(from->incomplete);
+    assert(to->incomplete);
+    // If these datatypes are part of the same SCC, there's nothing
+    // to do - they are contracted in the condensation.
+    if (get_scc_representative(from) == get_scc_representative(to)) {
+        // Nothing to do
+        return;
+    }
+    // See if adding this edge would result in a cycle.
+    jl_array_t *cycle = find_new_cycle(get_scc_representative(from),
+                                        get_scc_representative(to));
+    if (cycle) {
+        // If so, we need to contract the cycle by merging all SCCs that
+        // are part of the cycle. Arbitrarily pick the first SCC as the new
+        // SCC representative.
+        jl_datatype_t *new_scc_rep = (jl_datatype_t*)jl_array_ptr_ref(cycle, 0);
+        // First go through and set the elements to be members of the
+        // new cycle.
+        for (int i = 1; i < jl_array_len(cycle); ++i) {
+            jl_datatype_t *old_scc = (jl_datatype_t*)jl_array_ptr_ref(cycle, i);
+            old_scc->scc = new_scc_rep;
+        }
+        // Now go through and merge the various array lists
+        filter_scc(new_scc_rep->depends, new_scc_rep);
+        filter_scc(new_scc_rep->dependents, new_scc_rep);
+        for (int i = 1; i < jl_array_len(cycle); ++i) {
+            jl_datatype_t *old_scc = (jl_datatype_t*)jl_array_ptr_ref(cycle, 1);
+            filter_cycle_and_append(new_scc_rep->depends, old_scc->depends, new_scc_rep);
+            filter_cycle_and_append(new_scc_rep->dependents, old_scc->dependents, new_scc_rep);
+        }
+        return;
+    }
+    // Check if we need to merge cycle
+    jl_array_ptr_1d_push(get_scc_representative(to)->depends,
+        (jl_value_t*)get_scc_representative(from));
+    jl_array_ptr_1d_push(get_scc_representative(from)->dependents,
+        (jl_value_t*)get_scc_representative(to));
+}
+
+ static void finish_scc(jl_module_t *mod, jl_datatype_t *scc) {
+    assert(jl_array_len(scc->depends) == 0);
+    arraylist_t worklist;
+    arraylist_new(&worklist, 0);
+    arraylist_push(&worklist, scc);
+    while (worklist.len != 0) {
+        jl_datatype_t *dt = (jl_datatype_t*)arraylist_pop(&worklist);
+        if (!dt->incomplete)
+            continue;
+        dt->incomplete = 0;
+        // We don't store the members of the SCC. Instead, do a search through
+        // the various things that generate dependencies.
+        for (int i = 0; i < jl_svec_len(scc->types); ++i) {
+            jl_datatype_t *fdt = (jl_datatype_t*)jl_svecref(scc->types, i);
+            if (fdt->incomplete && get_scc_representative(fdt) == scc)
+                arraylist_push(&worklist, fdt);
+        }
+        jl_compute_field_offsets(dt);
+    }
+    scc->depends = NULL;
+    scc->dependents = NULL;
+    // See if there were any delayed method instantiations that we now need to
+    // add to the method table.
+    for (size_t i = 0; i < mod->deferred.len; ++i) {
+        jl_svec_t *item = (jl_svec_t*)mod->deferred.items[i];
+        jl_method_t *m = (jl_method_t*)jl_svecref(item, 1);
+        if (((jl_datatype_t*)jl_unwrap_unionall(m->sig))->scc == scc) {
+            jl_method_table_insert(
+                (jl_methtable_t*)jl_svecref(item, 0), m, NULL);
+        }
+    }
+}
+
+extern jl_array_t *jl_all_methods;
+void dt_mark_incomplete(jl_datatype_t *dt, jl_value_t *v)
+{
+    // While we don't have GC hooked up, just push these into a global root;
+    if (jl_all_methods == NULL)
+        jl_all_methods = jl_alloc_vec_any(0);
+    if (dt->incomplete == 0) {
+        dt->scc = dt;
+        dt->depends = jl_alloc_vec_any(0);
+        jl_array_ptr_1d_push(jl_all_methods, (jl_value_t*)dt->depends);
+        dt->dependents = jl_alloc_vec_any(0);
+        jl_array_ptr_1d_push(jl_all_methods, (jl_value_t*)dt->dependents);
+    }
+    if (jl_typeis(v, jl_placeholder_type)) {
+        jl_array_ptr_1d_push(
+                ((jl_placeholder_t*)v)->dependents,
+                (jl_value_t*)dt);
+        jl_array_ptr_1d_push(dt->depends, v);
+    } else if (jl_is_datatype(v)) {
+        assert(((jl_datatype_t*)v)->incomplete);
+        jl_array_ptr_1d_push(
+                ((jl_datatype_t*)v)->dependents,
+                (jl_value_t*)dt);
+        jl_array_ptr_1d_push(dt->depends, v);
+    }
+    dt->incomplete = 1;
+}
+
+
 static void eval_structtype(jl_expr_t *ex, interpreter_state *s)
 {
     jl_value_t **args = jl_array_ptr_data(ex->args);
@@ -261,7 +434,9 @@ static void eval_structtype(jl_expr_t *ex, interpreter_state *s)
         jl_gc_wb(dt, dt->types);
         for (size_t i = 0; i < jl_svec_len(dt->types); i++) {
             jl_value_t *elt = jl_svecref(dt->types, i);
-            if ((!jl_is_type(elt) && !jl_is_typevar(elt)) || jl_is_vararg_type(elt)) {
+            if (jl_typeis(elt, jl_placeholder_type)) {
+                dt_mark_incomplete(dt, elt);
+            } else if ((!jl_is_type(elt) && !jl_is_typevar(elt)) || jl_is_vararg_type(elt)) {
                 jl_type_error_rt(jl_symbol_name(dt->name->name),
                                  "type definition",
                                  (jl_value_t*)jl_type_type, elt);

src/jltypes.c

@@ -39,6 +39,7 @@ jl_typename_t *jl_vecelement_typename;
 jl_unionall_t *jl_vararg_type;
 jl_typename_t *jl_vararg_typename;
 jl_datatype_t *jl_tvar_type;
+jl_datatype_t *jl_placeholder_type;
 jl_datatype_t *jl_uniontype_type;
 jl_datatype_t *jl_unionall_type;
 jl_datatype_t *jl_datatype_type;
@@ -1322,6 +1323,24 @@ static jl_value_t *inst_datatype_inner(jl_datatype_t *dt, jl_svec_t *p, jl_value
         jl_compute_field_offsets(ndt);
     }
 
+    for (int i = 0; ndt->types && i < jl_svec_len(ndt->types); ++i) {
+        jl_value_t *v = jl_svecref(ndt->types, i);
+        if (jl_typeis(v, jl_placeholder_type)) {
+            dt_mark_incomplete(ndt, v);
+        } else if (jl_is_datatype(v) && ((jl_datatype_t*)v)->incomplete) {
+            dt_mark_incomplete(ndt, v);
+        }
+    }
+
+    for (int i = 0; ndt->parameters && i < jl_svec_len(ndt->parameters); ++i) {
+        jl_value_t *p = jl_svecref(ndt->parameters, i);
+        if (jl_typeis(p, jl_placeholder_type)) {
+            dt_mark_incomplete(ndt, p);
+        } else if (jl_is_datatype(p) && ((jl_datatype_t*)p)->incomplete) {
+            dt_mark_incomplete(ndt, p);
+        }
+    }
+
     if (istuple)
         ndt->ninitialized = ntp - isvatuple;
     else if (isnamedtuple)
@@ -1707,7 +1726,7 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_datatype_type->name->wrapper = (jl_value_t*)jl_datatype_type;
     jl_datatype_type->super = (jl_datatype_t*)jl_type_type;
     jl_datatype_type->parameters = jl_emptysvec;
-    jl_datatype_type->name->names = jl_perm_symsvec(21,
+    jl_datatype_type->name->names = jl_perm_symsvec(23,
                                                     "name",
                                                     "super",
                                                     "parameters",
@@ -1720,6 +1739,7 @@ void jl_init_types(void) JL_GC_DISABLED
                                                     "uid",
                                                     "abstract",
                                                     "mutable",
+                                                    "incomplete",
                                                     "hasfreetypevars",
                                                     "isconcretetype",
                                                     "isdispatchtuple",
@@ -1728,8 +1748,10 @@ void jl_init_types(void) JL_GC_DISABLED
                                                     "isinlinealloc",
                                                     "has_concrete_subtype",
                                                     "llvm::StructType",
-                                                    "llvm::DIType");
-    jl_datatype_type->types = jl_svec(21,
+                                                    "llvm::DIType",
+                                                    "padding");
+
+    jl_datatype_type->types = jl_svec(23,
                                       jl_typename_type,
                                       jl_datatype_type,
                                       jl_simplevector_type,
@@ -1738,7 +1760,8 @@ void jl_init_types(void) JL_GC_DISABLED
                                       jl_any_type, jl_any_type, jl_any_type, jl_any_type,
                                       jl_any_type, jl_any_type, jl_any_type, jl_any_type,
                                       jl_any_type, jl_any_type, jl_any_type, jl_any_type,
-                                      jl_any_type, jl_any_type, jl_any_type);
+                                      jl_any_type, jl_any_type, jl_any_type, jl_any_type,
+                                      jl_any_type);
     jl_datatype_type->instance = NULL;
     jl_datatype_type->uid = jl_assign_type_uid();
     jl_datatype_type->struct_decl = NULL;
@@ -2005,6 +2028,11 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_array_uint8_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint8_type, jl_box_long(1));
     jl_array_int32_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_int32_type, jl_box_long(1));
 
+    jl_placeholder_type = jl_new_datatype(jl_symbol("Placeholder"), core, jl_any_type, jl_emptysvec,
+                                   jl_perm_symsvec(1, "dependents"),
+                                   jl_svec(1, jl_array_any_type),
+                                   0, 1, 1);
+
     jl_expr_type =
         jl_new_datatype(jl_symbol("Expr"), core,
                         jl_any_type, jl_emptysvec,
@@ -2297,8 +2325,10 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_svecset(jl_datatype_type->types, 16, jl_bool_type);
     jl_svecset(jl_datatype_type->types, 17, jl_bool_type);
     jl_svecset(jl_datatype_type->types, 18, jl_bool_type);
-    jl_svecset(jl_datatype_type->types, 19, jl_voidpointer_type);
+    jl_svecset(jl_datatype_type->types, 19, jl_bool_type);
     jl_svecset(jl_datatype_type->types, 20, jl_voidpointer_type);
+    jl_svecset(jl_datatype_type->types, 21, jl_voidpointer_type);
+    jl_svecset(jl_datatype_type->types, 22, jl_voidpointer_type);
     jl_svecset(jl_typename_type->types, 1, jl_module_type);
     jl_svecset(jl_typename_type->types, 6, jl_long_type);
     jl_svecset(jl_typename_type->types, 3, jl_type_type);