Skip to content

Commit

Permalink
[generator] Add --with-javadoc-xml=FILE support.
Browse files Browse the repository at this point in the history
Commit 69e1b80 added `tools/java-source-utils`, which can parse Java
source code and extract Javadoc comments into an XML file:

	$ java -jar "java-source-utils.jar" -v \
		$HOME/android-toolchain/sdk/platforms/android-29/android-stubs-src.jar \
		--output-javadoc android-javadoc.xml

What can we *do* with the generated `android-javadoc.xml`?

`android-javadoc.xml` contains parameter names, and thus can be used
with `class-parse --docspath`; see commit 806082f.

What we *really* want to do is make the Javadoc information *useful*
to consumers of the binding assembly.  This means that we want a
C# XML Documentation file for the binding assembly.

The most straightforward way to get a C# XML Documentation File is to
emit [C# XML Documentation Comments][0] into the binding source code!

Add a new `generator --with-javadoc-xml=FILE` option.  When specified,
`FILE` will be treated as an XML file containing the output from
`java-source-utils.jar --output-javadoc` (see 69e1b80)`, and all
`<javadoc/>` elements within the XML file will be associated with C#
types and members to emit, based on the `//@jni-signature` and
`//@name` attributes, as appropriate.

When the bindings are written to disk, the Javadoc comments will be
translated to C# XML Documentation comments, in a "best effort" basis.
(THIS WILL BE INCOMPLETE.)

To perform the Javadoc-to-C# XML Documentation comments conversion,
add a new Irony-based grammar to `Java.Interop.Tools.JavaSource.dll`,
in the new `Java.Interop.Tools.JavaSource.SourceJavadocToXmldocParser`
type, which parses the Javadoc content and translates to XML.

TODO:

  * Properties?

[0]: https://docs.microsoft.com/en-us/dotnet/csharp/codedoc
  • Loading branch information
jonpryor committed Sep 25, 2020
1 parent ee7afee commit 6689716
Show file tree
Hide file tree
Showing 31 changed files with 1,198 additions and 1 deletion.
23 changes: 23 additions & 0 deletions src/Java.Interop.Tools.JavaSource/IronyExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Irony.Ast;
using Irony.Parsing;

namespace Java.Interop.Tools.JavaSource {

static class IronyExtensions {

public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm delimiter, BnfTerm of)
{
star.Rule = grammar.MakeStarRule (star, delimiter, of);
}

public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm of)
{
star.Rule = grammar.MakeStarRule (star, of);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Xml.Linq;

using Irony.Ast;
using Irony.Parsing;

namespace Java.Interop.Tools.JavaSource {

sealed class JavadocInfo {
public readonly ICollection<XNode> Parameters = new Collection<XNode> ();
public readonly ICollection<XNode> Paragraphs = new Collection<XNode> ();
public readonly ICollection<XNode> Exceptions = new Collection<XNode> ();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Xml.Linq;

using Irony.Ast;
using Irony.Parsing;

namespace Java.Interop.Tools.JavaSource {

public partial class SourceJavadocToXmldocGrammar {

// https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#javadoctags
public class BlockTagsBnfTerms {

internal BlockTagsBnfTerms ()
{
}

internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
{
AllBlockTerms.Rule = AuthorDeclaration
| ApiSinceDeclaration
| DeprecatedDeclaration
| DeprecatedSinceDeclaration
| ExceptionDeclaration
| ParamDeclaration
| ReturnDeclaration
| SeeDeclaration
| SerialDataDeclaration
| SerialFieldDeclaration
| SinceDeclaration
| ThrowsDeclaration
| VersionDeclaration
;
BlockValue.Rule = Cdata | grammar.InlineTagsTerms.AllInlineTerms;

AuthorDeclaration.Rule = "@author" + BlockValue;
AuthorDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
// Ignore; not sure how best to convert to Xmldoc
parseNode.AstNode = "";
};

ApiSinceDeclaration.Rule = "@apiSince" + BlockValue;
ApiSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = new XElement ("para", "Added in API level " + parseNode.ChildNodes [1].AstNode?.ToString () + ".");
};

DeprecatedDeclaration.Rule = "@deprecated" + BlockValue;
DeprecatedDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = new XElement ("para", "This member is deprecated. " + parseNode.ChildNodes [1].AstNode?.ToString ());
};

DeprecatedSinceDeclaration.Rule = "@deprecatedSince" + BlockValue;
DeprecatedSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = new XElement ("para", "This member was deprecated in API level " + parseNode.ChildNodes [1].AstNode?.ToString () + ".");
};

var nonSpaceTerm = new RegexBasedTerminal ("[^ ]", "[^ ]+") {
AstConfig = new AstNodeConfig {
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value,
},
};

ExceptionDeclaration.Rule = "@exception" + nonSpaceTerm + BlockValue;
ExceptionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
// TODO: convert `nonSpaceTerm` into a proper CREF
parseNode.AstNode = new XElement ("exception",
new XAttribute ("cref", parseNode.ChildNodes [1].AstNode?.ToString ()),
parseNode.ChildNodes [2].AstNode);
};

ParamDeclaration.Rule = "@param" + nonSpaceTerm + BlockValue;
ParamDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = new XElement ("param",
new XAttribute ("name", parseNode.ChildNodes [1].AstNode?.ToString ()),
parseNode.ChildNodes [2].AstNode);
};

ReturnDeclaration.Rule = "@return" + BlockValue;
ReturnDeclaration.Flags = TermFlags.IsMultiline;
ReturnDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
var returns = parseNode.ChildNodes [1].AstNode.ToString ();
parseNode.AstNode = new XElement ("returns", returns);
};

SeeDeclaration.Rule = "@see" + BlockValue;
SeeDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
// TODO: @see supports multiple forms; see: https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#see
parseNode.AstNode = new XElement ("altmember",
new XAttribute ("cref", parseNode.ChildNodes [1].AstNode?.ToString ()));
};

SinceDeclaration.Rule = "@since" + BlockValue;
SinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = new XElement ("para", "Added in " + parseNode.ChildNodes [1].AstNode?.ToString () + ".");
};

ThrowsDeclaration.Rule = "@throws" + nonSpaceTerm + BlockValue;
ThrowsDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
// TODO: convert `nonSpaceTerm` into a proper CREF
parseNode.AstNode = new XElement ("exception",
new XAttribute ("cref", parseNode.ChildNodes [1].AstNode?.ToString ()),
parseNode.ChildNodes [2].AstNode);
};

// Ignore serialization informatino
SerialDeclaration.Rule = "@serial" + BlockValue;
SerialDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = "";
};

SerialDataDeclaration.Rule = "@serialData" + BlockValue;
SerialDataDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = "";
};

SerialFieldDeclaration.Rule = "@serialField" + BlockValue;
SerialFieldDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = "";
};

// Ignore Version
VersionDeclaration.Rule = "@version" + BlockValue;
VersionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = "";
};
}

public readonly NonTerminal AllBlockTerms = new NonTerminal (nameof (AllBlockTerms), ConcatChildNodes);

public readonly Terminal Cdata = new RegexBasedTerminal (nameof (BlockValue), "[^<]*") {
AstConfig = new AstNodeConfig {
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
},
};

public readonly NonTerminal BlockValue = new NonTerminal (nameof (BlockValue), ConcatChildNodes);
public readonly NonTerminal AuthorDeclaration = new NonTerminal (nameof (AuthorDeclaration));
public readonly NonTerminal ApiSinceDeclaration = new NonTerminal (nameof (ApiSinceDeclaration));
public readonly NonTerminal DeprecatedDeclaration = new NonTerminal (nameof (DeprecatedDeclaration));
public readonly NonTerminal DeprecatedSinceDeclaration = new NonTerminal (nameof (DeprecatedSinceDeclaration));
public readonly NonTerminal ExceptionDeclaration = new NonTerminal (nameof (ExceptionDeclaration));
public readonly NonTerminal ParamDeclaration = new NonTerminal (nameof (ParamDeclaration));
public readonly NonTerminal ReturnDeclaration = new NonTerminal (nameof (ReturnDeclaration));
public readonly NonTerminal SeeDeclaration = new NonTerminal (nameof (SeeDeclaration));
public readonly NonTerminal SerialDeclaration = new NonTerminal (nameof (SerialDeclaration));
public readonly NonTerminal SerialDataDeclaration = new NonTerminal (nameof (SerialDataDeclaration));
public readonly NonTerminal SerialFieldDeclaration = new NonTerminal (nameof (SerialFieldDeclaration));
public readonly NonTerminal SinceDeclaration = new NonTerminal (nameof (SinceDeclaration));
public readonly NonTerminal ThrowsDeclaration = new NonTerminal (nameof (ThrowsDeclaration));
public readonly NonTerminal VersionDeclaration = new NonTerminal (nameof (VersionDeclaration));
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Xml.Linq;

using Irony.Ast;
using Irony.Parsing;

namespace Java.Interop.Tools.JavaSource {

using static IronyExtensions;

public partial class SourceJavadocToXmldocGrammar {

public class HtmlBnfTerms {
internal HtmlBnfTerms ()
{
}

internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
{
AllHtmlTerms.Rule = InlineDeclaration
| PBlockDeclaration
;

InlineDeclaration.Rule = ParsedCharacterData
| FontStyleDeclaration
/*
| PhraseDeclaration
| SpecialDeclaration
| FormCtrlDeclaration
*/
| grammar.InlineTagsTerms.AllInlineTerms
| UnknownHtmlElementStart
;
InlineDeclarations.MakeStarRule (grammar, InlineDeclaration);

var fontstyle_tt = new NonTerminal ("<tt>") {
Rule = CreateStartElement ("tt", grammar) + InlineDeclarations + CreateEndElement ("tt", grammar),
AstConfig = new AstNodeConfig {
NodeCreator = (context, parseNode) => {
parseNode.AstNode = new XElement ("c",
parseNode.ChildNodes
.Select (c => c.AstNode ?? ""));
},
},
};

var fontstyle_i = new NonTerminal ("<i>", ConcatChildNodes) {
Rule = CreateStartElement ("i", grammar) + InlineDeclarations + CreateEndElement ("i", grammar),
};

FontStyleDeclaration.Rule = fontstyle_tt | fontstyle_i;

PBlockDeclaration.Rule =
CreateStartElement ("p", grammar) + InlineDeclarations + CreateEndElement ("p", grammar, optional:true)
;
PBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
parseNode.AstNode = new XElement ("para",
parseNode.ChildNodes
.Select (c => c.AstNode ?? ""));
};
}

public readonly NonTerminal AllHtmlTerms = new NonTerminal (nameof (AllHtmlTerms), ConcatChildNodes);


// https://www.w3.org/TR/html401/struct/global.html#h-7.5.3
// public readonly Terminal ParsedCharacterData = new RegexBasedTerminal (nameof (ParsedCharacterData), "[^<{@}]*") {
// public readonly Terminal ParsedCharacterData = new WikiTextTerminal (nameof (ParsedCharacterData)) {*
public readonly Terminal ParsedCharacterData = new CharacterDataTerminal ("#PCDATA") {
// Priority = TerminalPriority.Low,
AstConfig = new AstNodeConfig {
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
},
};

// https://www.w3.org/TR/html4/sgml/dtd.html#inline
public readonly NonTerminal InlineDeclaration = new NonTerminal (nameof (InlineDeclaration), ConcatChildNodes);
public readonly NonTerminal InlineDeclarations = new NonTerminal (nameof (InlineDeclarations), ConcatChildNodes);
// https://www.w3.org/TR/html4/sgml/dtd.html#fontstyle
public readonly NonTerminal FontStyleDeclaration = new NonTerminal (nameof (FontStyleDeclaration), ConcatChildNodes);
// https://www.w3.org/TR/html4/sgml/dtd.html#phrase
public readonly NonTerminal PhraseDeclaration = new NonTerminal (nameof (PhraseDeclaration), ConcatChildNodes);
// https://www.w3.org/TR/html4/sgml/dtd.html#special
public readonly NonTerminal SpecialDeclaration = new NonTerminal (nameof (SpecialDeclaration), ConcatChildNodes);
// https://www.w3.org/TR/html4/sgml/dtd.html#formctrl
public readonly NonTerminal FormCtrlDeclaration = new NonTerminal (nameof (FormCtrlDeclaration), ConcatChildNodes);
// https://www.w3.org/TR/html4/sgml/dtd.html#block
public readonly NonTerminal BlockDeclaration = new NonTerminal (nameof (BlockDeclaration), ConcatChildNodes);
public readonly NonTerminal PBlockDeclaration = new NonTerminal (nameof (PBlockDeclaration), ConcatChildNodes);

public readonly Terminal UnknownHtmlElementStart = new UnknownHtmlElementStartTerminal (nameof (UnknownHtmlElementStart)) {
AstConfig = new AstNodeConfig {
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
},
};

static NonTerminal CreateStartElement (string startElement, Grammar grammar)
{
var start = new NonTerminal ("<" + startElement + ">", nodeCreator: (context, parseNode) => parseNode.AstNode = "") {
Rule = grammar.ToTerm ("<" + startElement + ">") | "<" + startElement.ToUpperInvariant () + ">",
};
return start;
}

static NonTerminal CreateEndElement (string endElement, Grammar grammar, bool optional = false)
{
var end = new NonTerminal (endElement, nodeCreator: (context, parseNode) => parseNode.AstNode = "") {
Rule = grammar.ToTerm ("</" + endElement + ">") | "</" + endElement.ToUpperInvariant () + ">",
};
if (optional) {
end.Rule |= grammar.Empty;
}
return end;
}
}
}

// Based in part on WikiTextTerminal
class CharacterDataTerminal : Terminal {

private char[] _stopChars;

public CharacterDataTerminal (string name)
: base (name)
{
base.Priority = TerminalPriority.Low;
}

public override void Init (GrammarData grammarData)
{
base.Init (grammarData);
var stopCharSet = new Irony.CharHashSet ();
foreach(var term in grammarData.Terminals) {
var firsts = term.GetFirsts ();
if (firsts == null)
continue;
foreach (var first in firsts) {
if (string.IsNullOrEmpty (first))
continue;
stopCharSet.Add (first [0]);
}
}
_stopChars = stopCharSet.ToArray();
}

public override Token TryMatch (ParsingContext context, ISourceStream source)
{
var stopIndex = source.Text.IndexOfAny (_stopChars, source.Location.Position);
if (stopIndex == source.Location.Position)
return null;
if (stopIndex < 0)
stopIndex = source.Text.Length;
source.PreviewPosition = stopIndex;

// preserve leading whitespace, if present.
int start = source.Location.Position;
while (start > 0 && char.IsWhiteSpace (source.Text, start-1)) {
start--;
}
var content = source.Text.Substring (start, stopIndex - start);

return source.CreateToken (this.OutputTerminal, content);
}
}

class UnknownHtmlElementStartTerminal : Terminal {

public UnknownHtmlElementStartTerminal (string name)
: base (name)
{
base.Priority = TerminalPriority.Low-1;
}

public override void Init (GrammarData grammarData)
{
base.Init (grammarData);
}

public override Token TryMatch (ParsingContext context, ISourceStream source)
{
if (source.Text [source.Location.Position] != '<')
return null;
source.PreviewPosition += 1;
return source.CreateToken (this.OutputTerminal, "<");
}
}
}
Loading

0 comments on commit 6689716

Please sign in to comment.