[RFC 50] add member docstrings (ponylang#2543)

* adapt parser to accept field docstrings

* fix desugaring of lambda captures as fields

it was failing during check_tree due to a missing optional STRING node for the docstring

* docgen: add docstring to fields docs

and display fields more prominently

* update pony.g for field docstrings

pony.g

@@ -30,7 +30,7 @@ members
   ;
 
 field
-  : ('var' | 'let' | 'embed') ID ':' type ('=' infix)?
+  : ('var' | 'let' | 'embed') ID ':' type ('=' infix)? STRING?
   ;
 
 method

src/libponyc/ast/parser.c

@@ -1156,6 +1156,7 @@ DEF(field);
   SKIP("mandatory type declaration on field", TK_COLON);
   RULE("field type", type);
   IF(TK_ASSIGN, RULE("field value", infix));
+  OPT TOKEN("docstring", TK_STRING);
   DONE();
 
 // {field} {method}

src/libponyc/ast/treecheckdef.h

@@ -66,7 +66,8 @@ RULE(field,
   HAS_TYPE(type)
   CHILD(id)
   CHILD(type, none)  // Field type
-  CHILD(expr, none),
+  CHILD(expr, none)
+  CHILD(string, none),
   TK_FLET, TK_FVAR, TK_EMBED);
 
 RULE(method,

src/libponyc/expr/lambda.c

@@ -131,7 +131,8 @@ static bool make_capture_field(pass_opt_t* opt, ast_t* capture,
     NODE(TK_FVAR,
       TREE(id_node)
       TREE(type)
-      TREE(value)));
+      TREE(value)
+      NONE));
 
   *out_field = field;
   return true;
@@ -512,7 +513,8 @@ static bool capture_from_reference(pass_opt_t* opt, ast_t* ctx, ast_t* ast,
     NODE(TK_FVAR,
       ID(name)
       TREE(type)
-      NODE(TK_REFERENCE, ID(name))));
+      NODE(TK_REFERENCE, ID(name))
+      NONE));
 
   ast_list_append(captures, last_capture, field);
   return true;

src/libponyc/pass/docgen.c

@@ -515,8 +515,8 @@ static void add_source_code_link(docgen_t* docgen, ast_t* elem)
 
   if (doc_source != NULL) {
       fprintf(
-        docgen->type_file, 
-        "\n<span class=\"source-link\">[[Source]](%s#L%zd)</span>\n", 
+        docgen->type_file,
+        "\n<span class=\"source-link\">[[Source]](%s#L%zd)</span>\n",
         doc_source->doc_path, ast_line(elem)
       );
   } else {
@@ -539,15 +539,15 @@ static void doc_fields(docgen_t* docgen, docgen_opt_t* docgen_opt,
 
   if(fields->next == NULL)  // No fields
     return;
-  
+
   fprintf(docgen->type_file, "## %s\n\n", title);
 
   for(ast_list_t* p = fields->next; p != NULL; p = p->next)
   {
     ast_t* field = p->ast;
     pony_assert(field != NULL);
 
-    AST_GET_CHILDREN(field, id, type, init);
+    AST_GET_CHILDREN(field, id, type, init, doc);
     const char* name = ast_name(id);
     pony_assert(name != NULL);
 
@@ -562,9 +562,14 @@ static void doc_fields(docgen_t* docgen, docgen_opt_t* docgen_opt,
       default: pony_assert(0);
     }
 
-    fprintf(docgen->type_file, "* %s %s: ", ftype, name);
+    fprintf(docgen->type_file, "##### %s %s: ", ftype, name);
     doc_type(docgen, docgen_opt, type, true, true);
     add_source_code_link(docgen, field);
+    fprintf(docgen->type_file, "\n");
+
+    if(ast_id(doc) != TK_NONE)
+      fprintf(docgen->type_file, "%s\n\n", ast_name(doc));
+
     fprintf(docgen->type_file, "\n\n---\n\n");
   }
 }
@@ -833,7 +838,7 @@ static doc_sources_t* copy_source_to_doc_src(docgen_t* docgen, source_t* source,
 
   //Create directory for [documentationDir]/src/[package_name]
   pony_mkdir(source_dir);
-  
+
   // Get absolute path for [documentationDir]/src/[package_name]/[filename].md
   size_t file_path_alloc_size = FILENAME_MAX;
   char* path = (char*) ponyint_pool_alloc_size(FILENAME_MAX);
@@ -848,7 +853,7 @@ static doc_sources_t* copy_source_to_doc_src(docgen_t* docgen, source_t* source,
   size_t doc_source_dir_relative_alloc_size = 0;
   doc_source_dir_relative = concat(doc_source_dir_relative, "/", &doc_source_dir_relative_alloc_size);
   ponyint_pool_free_size(old_ptr_alloc_size, (void*) old_ptr);
-  
+
   // Get relative path for [documentationDir]/src/[package_name]/[filename].md
   size_t doc_path_alloc_size = 0;
   const char* doc_path = concat(doc_source_dir_relative, filename_md_extension, &doc_path_alloc_size);
@@ -912,10 +917,10 @@ static char* replace_path_separator(const char* path, size_t* name_len) {
 }
 
 static void include_source_if_needed(
-  docgen_t* docgen, 
-  source_t* source, 
+  docgen_t* docgen,
+  source_t* source,
   const char* package_name
-) 
+)
 {
   pony_assert(source != NULL);
   pony_assert(docgen != NULL);
@@ -1012,7 +1017,8 @@ static void doc_entity(docgen_t* docgen, docgen_opt_t* docgen_opt, ast_t* ast)
   add_source_code_link(docgen, ast);
 
   if(ast_id(doc) != TK_NONE)
-    fprintf(docgen->type_file, "%s\n\n", ast_name(doc));
+    // additional linebreak for better source code link display with docstring
+    fprintf(docgen->type_file, "\n%s\n\n", ast_name(doc));
 
   // code block
   fprintf(docgen->type_file, "```pony\n");
@@ -1387,7 +1393,7 @@ void generate_docs(ast_t* program, pass_opt_t* options)
       ponyint_pool_free_size(sizeof(doc_sources_t), (void*) current_source_ptr_copy);
     }
   }
-  
+
   // Tidy up
   if(docgen.index_file != NULL)
     fclose(docgen.index_file);

test/libponyc/parse_entity.cc

@@ -113,6 +113,7 @@ TEST_F(ParseEntityTest, ClassMaximal)
     "class box Foo[A] is T"
     "  \"Doc\""
     "  let f1:T1"
+    "  \"member Doc\""
     "  let f2:T2 = 5"
     "  var f3:P3.T3"
     "  var f4:T4 = 9"
@@ -656,65 +657,28 @@ TEST_F(ParseEntityTest, FieldMustHaveType)
   TEST_ERROR(src);
 }
 
-
-TEST_F(ParseEntityTest, LetFieldMustHaveType)
-{
-  const char* src = "class Foo let bar";
-
-  TEST_ERROR(src);
-}
-
-
-TEST_F(ParseEntityTest, FieldDelegateCannotBeUnion)
-{
-  const char* src =
-    "trait T\n"
-    "trait T1\n"
-    "trait T2\n"
-
-    "class Foo\n"
-    "  var bar: T delegate (T1 | T2)";
-
-  TEST_ERROR(src);
-}
-
-
-TEST_F(ParseEntityTest, FieldDelegateCannotBeTuple)
-{
-  const char* src =
-    "trait T\n"
-    "trait T1\n"
-    "trait T2\n"
-
-    "class Foo\n"
-    "  var bar: T delegate (T1, T2)";
-
-  TEST_ERROR(src);
-}
-
-
-TEST_F(ParseEntityTest, FieldDelegateCannotBeArrow)
+TEST_F(ParseEntityTest, FieldCanHaveADocString)
 {
   const char* src =
-    "trait T\n"
-    "trait T1\n"
-    "trait T2\n"
-
-    "class Foo\n"
-    "  var bar: T delegate T1->T2";
-
-  TEST_ERROR(src);
+    "class Foo"
+    "  var baz: U8 "
+    "  \"field docstring\""
+    "  "
+    "  let bool: Bool = true"
+    "  \"\"\""
+    "  multiline"
+    "  field"
+    "  docstring"
+    "  \"\"\""
+    "  embed data: Array[U8] = data.create(0)"
+    "  \"\"\"embed field docstring\"\"\"";
+  TEST_COMPILE(src);
 }
 
 
-TEST_F(ParseEntityTest, FieldDelegateCannotHaveCapability)
+TEST_F(ParseEntityTest, LetFieldMustHaveType)
 {
-  const char* src =
-    "trait T\n"
-    "trait T1\n"
-
-    "class Foo\n"
-    "  var bar: T delegate T1 ref";
+  const char* src = "class Foo let bar";
 
   TEST_ERROR(src);
 }