[generator] Add --with-javadoc-xml=FILE support.

Commit 69e1b80 added `tools/java-source-utils`, which can parse Java
source code and extract Javadoc comments into an XML file:

	$ java -jar "java-source-utils.jar" -v \
		$HOME/android-toolchain/sdk/platforms/android-29/android-stubs-src.jar \
		--output-javadoc android-javadoc.xml

What can we *do* with the generated `android-javadoc.xml`?

`android-javadoc.xml` contains parameter names, and thus can be used
with `class-parse --docspath`; see commit 806082f.

What we *really* want to do is make the Javadoc information *useful*
to consumers of the binding assembly.  This means that we want a
C# XML Documentation file for the binding assembly.

The most straightforward way to get a C# XML Documentation File is to
emit [C# XML Documentation Comments][0] into the binding source code!

Add a new `generator --with-javadoc-xml=FILE` option.  When specified,
`FILE` will be treated as an XML file containing the output from
`java-source-utils.jar --output-javadoc` (see 69e1b80)`, and all
`<javadoc/>` elements within the XML file will be associated with C#
types and members to emit, based on the `//@jni-signature` and
`//@name` attributes, as appropriate.

When the bindings are written to disk, the Javadoc comments will be
translated to C# XML Documentation comments, in a "best effort" basis.
(THIS WILL BE INCOMPLETE.)

To perform the Javadoc-to-C# XML Documentation comments conversion,
add a new Irony-based grammar to `Java.Interop.Tools.JavaSource.dll`,
in the new `Java.Interop.Tools.JavaSource.SourceJavadocToXmldocParser`
type, which parses the Javadoc content and translates to XML.

TODO:

  * Properties?

[0]: https://docs.microsoft.com/en-us/dotnet/csharp/codedoc

src/Java.Interop.Tools.JavaSource/IronyExtensions.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Irony.Ast;
+using Irony.Parsing;
+
+namespace Java.Interop.Tools.JavaSource {
+
+	static class IronyExtensions {
+
+		public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm delimiter, BnfTerm of)
+		{
+			star.Rule = grammar.MakeStarRule (star, delimiter, of);
+		}
+
+		public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm of)
+		{
+			star.Rule = grammar.MakeStarRule (star, of);
+		}
+	}
+}

src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/JavadocInfo.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.IO;
+using System.Linq;
+using System.Xml.Linq;
+
+using Irony.Ast;
+using Irony.Parsing;
+
+namespace Java.Interop.Tools.JavaSource {
+
+	sealed class JavadocInfo {
+		public  readonly    ICollection<XNode>  Parameters  = new Collection<XNode> ();
+		public  readonly    ICollection<XNode>  Paragraphs  = new Collection<XNode> ();
+		public  readonly    ICollection<XNode>  Exceptions  = new Collection<XNode> ();
+	}
+}

src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs

@@ -0,0 +1,156 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Xml.Linq;
+
+using Irony.Ast;
+using Irony.Parsing;
+
+namespace Java.Interop.Tools.JavaSource {
+
+	public partial class SourceJavadocToXmldocGrammar {
+
+		// https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#javadoctags
+		public class BlockTagsBnfTerms {
+
+			internal BlockTagsBnfTerms ()
+			{
+			}
+
+			internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
+			{
+				AllBlockTerms.Rule = AuthorDeclaration
+					| ApiSinceDeclaration
+					| DeprecatedDeclaration
+					| DeprecatedSinceDeclaration
+					| ExceptionDeclaration
+					| ParamDeclaration
+					| ReturnDeclaration
+					| SeeDeclaration
+					| SerialDataDeclaration
+					| SerialFieldDeclaration
+					| SinceDeclaration
+					| ThrowsDeclaration
+					| VersionDeclaration
+					;
+				BlockValue.Rule = Cdata | grammar.InlineTagsTerms.AllInlineTerms;
+
+				AuthorDeclaration.Rule = "@author" + BlockValue;
+				AuthorDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					// Ignore; not sure how best to convert to Xmldoc
+					parseNode.AstNode = "";
+				};
+
+				ApiSinceDeclaration.Rule = "@apiSince" + BlockValue;
+				ApiSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = new XElement ("para", "Added in API level " + parseNode.ChildNodes [1].AstNode?.ToString () + ".");
+				};
+
+				DeprecatedDeclaration.Rule = "@deprecated" + BlockValue;
+				DeprecatedDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = new XElement ("para", "This member is deprecated. " + parseNode.ChildNodes [1].AstNode?.ToString ());
+				};
+
+				DeprecatedSinceDeclaration.Rule = "@deprecatedSince" + BlockValue;
+				DeprecatedSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = new XElement ("para", "This member was deprecated in API level " + parseNode.ChildNodes [1].AstNode?.ToString () + ".");
+				};
+
+				var nonSpaceTerm = new RegexBasedTerminal ("[^ ]", "[^ ]+") {
+					AstConfig = new AstNodeConfig {
+						NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value,
+					},
+				};
+
+				ExceptionDeclaration.Rule = "@exception" + nonSpaceTerm + BlockValue;
+				ExceptionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					// TODO: convert `nonSpaceTerm` into a proper CREF
+					parseNode.AstNode = new XElement ("exception",
+							new XAttribute ("cref", parseNode.ChildNodes [1].AstNode?.ToString ()),
+							parseNode.ChildNodes [2].AstNode);
+				};
+
+				ParamDeclaration.Rule = "@param" + nonSpaceTerm + BlockValue;
+				ParamDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = new XElement ("param",
+							new XAttribute ("name", parseNode.ChildNodes [1].AstNode?.ToString ()),
+							parseNode.ChildNodes [2].AstNode);
+				};
+
+				ReturnDeclaration.Rule = "@return" + BlockValue;
+				ReturnDeclaration.Flags = TermFlags.IsMultiline;
+				ReturnDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					var returns = parseNode.ChildNodes [1].AstNode.ToString ();
+					parseNode.AstNode = new XElement ("returns", returns);
+				};
+
+				SeeDeclaration.Rule = "@see" + BlockValue;
+				SeeDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					// TODO: @see supports multiple forms; see: https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#see
+					parseNode.AstNode = new XElement ("altmember",
+							new XAttribute ("cref", parseNode.ChildNodes [1].AstNode?.ToString ()));
+				};
+
+				SinceDeclaration.Rule = "@since" + BlockValue;
+				SinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = new XElement ("para", "Added in " + parseNode.ChildNodes [1].AstNode?.ToString () + ".");
+				};
+
+				ThrowsDeclaration.Rule = "@throws" + nonSpaceTerm + BlockValue;
+				ThrowsDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					// TODO: convert `nonSpaceTerm` into a proper CREF
+					parseNode.AstNode = new XElement ("exception",
+							new XAttribute ("cref", parseNode.ChildNodes [1].AstNode?.ToString ()),
+							parseNode.ChildNodes [2].AstNode);
+				};
+
+				// Ignore serialization informatino
+				SerialDeclaration.Rule = "@serial" + BlockValue;
+				SerialDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = "";
+				};
+
+				SerialDataDeclaration.Rule = "@serialData" + BlockValue;
+				SerialDataDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = "";
+				};
+
+				SerialFieldDeclaration.Rule = "@serialField" + BlockValue;
+				SerialFieldDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = "";
+				};
+
+				// Ignore Version
+				VersionDeclaration.Rule = "@version" + BlockValue;
+				VersionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = "";
+				};
+			}
+
+			public  readonly    NonTerminal AllBlockTerms              = new NonTerminal (nameof (AllBlockTerms), ConcatChildNodes);
+
+			public  readonly    Terminal    Cdata                      = new RegexBasedTerminal (nameof (BlockValue), "[^<]*") {
+				AstConfig   = new AstNodeConfig {
+					NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
+				},
+			};
+
+			public  readonly    NonTerminal BlockValue                 = new NonTerminal (nameof (BlockValue), ConcatChildNodes);
+			public  readonly    NonTerminal AuthorDeclaration          = new NonTerminal (nameof (AuthorDeclaration));
+			public  readonly    NonTerminal ApiSinceDeclaration        = new NonTerminal (nameof (ApiSinceDeclaration));
+			public  readonly    NonTerminal DeprecatedDeclaration      = new NonTerminal (nameof (DeprecatedDeclaration));
+			public  readonly    NonTerminal DeprecatedSinceDeclaration = new NonTerminal (nameof (DeprecatedSinceDeclaration));
+			public  readonly    NonTerminal ExceptionDeclaration       = new NonTerminal (nameof (ExceptionDeclaration));
+			public  readonly    NonTerminal ParamDeclaration           = new NonTerminal (nameof (ParamDeclaration));
+			public  readonly    NonTerminal ReturnDeclaration          = new NonTerminal (nameof (ReturnDeclaration));
+			public  readonly    NonTerminal SeeDeclaration             = new NonTerminal (nameof (SeeDeclaration));
+			public  readonly    NonTerminal SerialDeclaration          = new NonTerminal (nameof (SerialDeclaration));
+			public  readonly    NonTerminal SerialDataDeclaration      = new NonTerminal (nameof (SerialDataDeclaration));
+			public  readonly    NonTerminal SerialFieldDeclaration     = new NonTerminal (nameof (SerialFieldDeclaration));
+			public  readonly    NonTerminal SinceDeclaration           = new NonTerminal (nameof (SinceDeclaration));
+			public  readonly    NonTerminal ThrowsDeclaration          = new NonTerminal (nameof (ThrowsDeclaration));
+			public  readonly    NonTerminal VersionDeclaration         = new NonTerminal (nameof (VersionDeclaration));
+		}
+	}
+}

src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs

@@ -0,0 +1,190 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Xml.Linq;
+
+using Irony.Ast;
+using Irony.Parsing;
+
+namespace Java.Interop.Tools.JavaSource {
+
+	using static IronyExtensions;
+
+	public partial class SourceJavadocToXmldocGrammar {
+
+		public class HtmlBnfTerms {
+			internal HtmlBnfTerms ()
+			{
+			}
+
+			internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
+			{
+				AllHtmlTerms.Rule = InlineDeclaration
+					| PBlockDeclaration
+					;
+
+				InlineDeclaration.Rule = ParsedCharacterData
+					| FontStyleDeclaration
+					/*
+					| PhraseDeclaration
+					| SpecialDeclaration
+					| FormCtrlDeclaration
+					*/
+					| grammar.InlineTagsTerms.AllInlineTerms
+					| UnknownHtmlElementStart
+					;
+				InlineDeclarations.MakeStarRule (grammar, InlineDeclaration);
+
+				var fontstyle_tt = new NonTerminal ("<tt>") {
+					Rule = CreateStartElement ("tt", grammar) + InlineDeclarations + CreateEndElement ("tt", grammar),
+					AstConfig   = new AstNodeConfig {
+						NodeCreator = (context, parseNode) => {
+							parseNode.AstNode = new XElement ("c",
+								parseNode.ChildNodes
+								.Select (c => c.AstNode ?? ""));
+						},
+					},
+				};
+
+				var fontstyle_i = new NonTerminal ("<i>", ConcatChildNodes) {
+					Rule = CreateStartElement ("i", grammar) + InlineDeclarations + CreateEndElement ("i", grammar),
+				};
+
+				FontStyleDeclaration.Rule = fontstyle_tt | fontstyle_i;
+
+				PBlockDeclaration.Rule =
+					CreateStartElement ("p", grammar) + InlineDeclarations + CreateEndElement ("p", grammar, optional:true)
+					;
+				PBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = new XElement ("para",
+							parseNode.ChildNodes
+							.Select (c => c.AstNode ?? ""));
+				};
+			}
+
+			public  readonly    NonTerminal AllHtmlTerms               = new NonTerminal (nameof (AllHtmlTerms), ConcatChildNodes);
+
+
+			// https://www.w3.org/TR/html401/struct/global.html#h-7.5.3
+//			public  readonly    Terminal    ParsedCharacterData        = new RegexBasedTerminal (nameof (ParsedCharacterData), "[^<{@}]*") {
+//			public  readonly    Terminal    ParsedCharacterData        = new WikiTextTerminal (nameof (ParsedCharacterData)) {*
+			public  readonly    Terminal    ParsedCharacterData        = new CharacterDataTerminal ("#PCDATA") {
+//				Priority    = TerminalPriority.Low,
+				AstConfig   = new AstNodeConfig {
+					NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
+				},
+			};
+
+			// https://www.w3.org/TR/html4/sgml/dtd.html#inline
+			public  readonly    NonTerminal InlineDeclaration           = new NonTerminal (nameof (InlineDeclaration), ConcatChildNodes);
+			public  readonly    NonTerminal InlineDeclarations          = new NonTerminal (nameof (InlineDeclarations), ConcatChildNodes);
+			// https://www.w3.org/TR/html4/sgml/dtd.html#fontstyle
+			public  readonly    NonTerminal FontStyleDeclaration        = new NonTerminal (nameof (FontStyleDeclaration), ConcatChildNodes);
+			// https://www.w3.org/TR/html4/sgml/dtd.html#phrase
+			public  readonly    NonTerminal PhraseDeclaration           = new NonTerminal (nameof (PhraseDeclaration), ConcatChildNodes);
+			// https://www.w3.org/TR/html4/sgml/dtd.html#special
+			public  readonly    NonTerminal SpecialDeclaration          = new NonTerminal (nameof (SpecialDeclaration), ConcatChildNodes);
+			// https://www.w3.org/TR/html4/sgml/dtd.html#formctrl
+			public  readonly    NonTerminal FormCtrlDeclaration         = new NonTerminal (nameof (FormCtrlDeclaration), ConcatChildNodes);
+			// https://www.w3.org/TR/html4/sgml/dtd.html#block
+			public  readonly    NonTerminal BlockDeclaration            = new NonTerminal (nameof (BlockDeclaration), ConcatChildNodes);
+			public  readonly    NonTerminal PBlockDeclaration           = new NonTerminal (nameof (PBlockDeclaration), ConcatChildNodes);
+
+			public  readonly    Terminal    UnknownHtmlElementStart     = new UnknownHtmlElementStartTerminal (nameof (UnknownHtmlElementStart)) {
+				AstConfig   = new AstNodeConfig {
+					NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
+				},
+			};
+
+			static NonTerminal CreateStartElement (string startElement, Grammar grammar)
+			{
+				var start   = new NonTerminal ("<" + startElement + ">", nodeCreator: (context, parseNode) => parseNode.AstNode = "") {
+					Rule    = grammar.ToTerm ("<" + startElement + ">") | "<" + startElement.ToUpperInvariant () + ">",
+				};
+				return start;
+			}
+
+			static NonTerminal CreateEndElement (string endElement, Grammar grammar, bool optional = false)
+			{
+				var end	    = new NonTerminal (endElement, nodeCreator: (context, parseNode) => parseNode.AstNode = "") {
+					Rule    = grammar.ToTerm ("</" + endElement + ">") | "</" + endElement.ToUpperInvariant () + ">",
+				};
+				if (optional) {
+					end.Rule |= grammar.Empty;
+				}
+				return end;
+			}
+		}
+	}
+
+	// Based in part on WikiTextTerminal
+	class CharacterDataTerminal : Terminal {
+
+		private char[] _stopChars;
+
+		public CharacterDataTerminal (string name)
+			: base (name)
+		{
+			base.Priority = TerminalPriority.Low;
+		}
+
+		public override void Init (GrammarData grammarData)
+		{
+			base.Init (grammarData);
+			var stopCharSet = new Irony.CharHashSet ();
+			foreach(var term in grammarData.Terminals) {
+				var firsts = term.GetFirsts ();
+				if (firsts == null)
+					continue;
+				foreach (var first in firsts) {
+					if (string.IsNullOrEmpty (first))
+						continue;
+					stopCharSet.Add (first [0]);
+				}
+			}
+			_stopChars = stopCharSet.ToArray(); 
+		}
+
+		public override Token TryMatch (ParsingContext context, ISourceStream source)
+		{
+			var stopIndex = source.Text.IndexOfAny (_stopChars, source.Location.Position);
+			if (stopIndex == source.Location.Position)
+				return null;
+			if (stopIndex < 0)
+				stopIndex = source.Text.Length;
+			source.PreviewPosition = stopIndex;
+
+			// preserve leading whitespace, if present.
+			int start = source.Location.Position;
+			while (start > 0 && char.IsWhiteSpace (source.Text, start-1)) {
+				start--;
+			}
+			var content = source.Text.Substring (start, stopIndex - start);
+
+			return source.CreateToken (this.OutputTerminal, content);
+		}
+	}
+
+	class UnknownHtmlElementStartTerminal : Terminal {
+
+		public UnknownHtmlElementStartTerminal (string name)
+			: base (name)
+		{
+			base.Priority = TerminalPriority.Low-1;
+		}
+
+		public override void Init (GrammarData grammarData)
+		{
+			base.Init (grammarData);
+		}
+
+		public override Token TryMatch (ParsingContext context, ISourceStream source)
+		{
+			if (source.Text [source.Location.Position] != '<')
+				return null;
+			source.PreviewPosition += 1;
+			return source.CreateToken (this.OutputTerminal, "<");
+		}
+	}
+}