org.assertj
diff --git a/src/main/java/us/codecraft/xsoup/XTokenQueue.java b/src/main/java/us/codecraft/xsoup/XTokenQueue.java
index 4fe5866..64f9072 100644
--- a/src/main/java/us/codecraft/xsoup/XTokenQueue.java
+++ b/src/main/java/us/codecraft/xsoup/XTokenQueue.java
@@ -1,6 +1,6 @@
package us.codecraft.xsoup;
-import org.jsoup.helper.StringUtil;
+import org.jsoup.internal.StringUtil;
import org.jsoup.helper.Validate;
import java.util.ArrayList;
diff --git a/src/main/java/us/codecraft/xsoup/xevaluator/CombiningEvaluator.java b/src/main/java/us/codecraft/xsoup/xevaluator/CombiningEvaluator.java
index 1d39e21..1946df1 100644
--- a/src/main/java/us/codecraft/xsoup/xevaluator/CombiningEvaluator.java
+++ b/src/main/java/us/codecraft/xsoup/xevaluator/CombiningEvaluator.java
@@ -1,6 +1,6 @@
package us.codecraft.xsoup.xevaluator;
-import org.jsoup.helper.StringUtil;
+import org.jsoup.internal.StringUtil;
import org.jsoup.nodes.Element;
import org.jsoup.select.Evaluator;
diff --git a/src/main/java/us/codecraft/xsoup/xevaluator/ElementOperator.java b/src/main/java/us/codecraft/xsoup/xevaluator/ElementOperator.java
index 777f3ed..9c9c8f0 100644
--- a/src/main/java/us/codecraft/xsoup/xevaluator/ElementOperator.java
+++ b/src/main/java/us/codecraft/xsoup/xevaluator/ElementOperator.java
@@ -1,6 +1,5 @@
package us.codecraft.xsoup.xevaluator;
-import org.jsoup.examples.HtmlToPlainText;
import org.jsoup.helper.Validate;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
diff --git a/src/main/java/us/codecraft/xsoup/xevaluator/HtmlToPlainText.java b/src/main/java/us/codecraft/xsoup/xevaluator/HtmlToPlainText.java
new file mode 100644
index 0000000..cfe9688
--- /dev/null
+++ b/src/main/java/us/codecraft/xsoup/xevaluator/HtmlToPlainText.java
@@ -0,0 +1,130 @@
+package us.codecraft.xsoup.xevaluator;
+
+import org.jsoup.Jsoup;
+import org.jsoup.internal.StringUtil;
+import org.jsoup.helper.Validate;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.nodes.Node;
+import org.jsoup.nodes.TextNode;
+import org.jsoup.select.Elements;
+import org.jsoup.select.NodeTraversor;
+import org.jsoup.select.NodeVisitor;
+
+import java.io.IOException;
+
+/**
+ * HTML to plain-text. This example program demonstrates the use of jsoup to convert HTML input to lightly-formatted
+ * plain-text. That is divergent from the general goal of jsoup's .text() methods, which is to get clean data from a
+ * scrape.
+ *
+ * Note that this is a fairly simplistic formatter -- for real world use you'll want to embrace and extend.
+ *
+ *
+ * To invoke from the command line, assuming you've downloaded the jsoup jar to your current directory:
+ * java -cp jsoup.jar org.jsoup.examples.HtmlToPlainText url [selector]
+ * where url is the URL to fetch, and selector is an optional CSS selector.
+ *
+ * @author Jonathan Hedley, jonathan@hedley.net
+ */
+public class HtmlToPlainText {
+ private static final String userAgent = "Mozilla/5.0 (jsoup)";
+ private static final int timeout = 5 * 1000;
+
+ public static void main(String... args) throws IOException {
+ Validate.isTrue(args.length == 1 || args.length == 2, "usage: java -cp jsoup.jar org.jsoup.examples.HtmlToPlainText url [selector]");
+ final String url = args[0];
+ final String selector = args.length == 2 ? args[1] : null;
+
+ // fetch the specified URL and parse to a HTML DOM
+ Document doc = Jsoup.connect(url).userAgent(userAgent).timeout(timeout).get();
+
+ HtmlToPlainText formatter = new HtmlToPlainText();
+
+ if (selector != null) {
+ Elements elements = doc.select(selector); // get each element that matches the CSS selector
+ for (Element element : elements) {
+ String plainText = formatter.getPlainText(element); // format that element to plain text
+ System.out.println(plainText);
+ }
+ } else { // format the whole doc
+ String plainText = formatter.getPlainText(doc);
+ System.out.println(plainText);
+ }
+ }
+
+ /**
+ * Format an Element to plain-text
+ * @param element the root element to format
+ * @return formatted text
+ */
+ public String getPlainText(Element element) {
+ FormattingVisitor formatter = new FormattingVisitor();
+ NodeTraversor.traverse(formatter, element); // walk the DOM, and call .head() and .tail() for each node
+
+ return formatter.toString();
+ }
+
+ // the formatting rules, implemented in a breadth-first DOM traverse
+ private static class FormattingVisitor implements NodeVisitor {
+ private static final int maxWidth = 80;
+ private int width = 0;
+ private StringBuilder accum = new StringBuilder(); // holds the accumulated text
+
+ // hit when the node is first seen
+ public void head(Node node, int depth) {
+ String name = node.nodeName();
+ if (node instanceof TextNode)
+ append(((TextNode) node).text()); // TextNodes carry all user-readable text in the DOM.
+ else if (name.equals("li"))
+ append("\n * ");
+ else if (name.equals("dt"))
+ append(" ");
+ else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5", "tr"))
+ append("\n");
+ }
+
+ // hit when all of the node's children (if any) have been visited
+ public void tail(Node node, int depth) {
+ String name = node.nodeName();
+ if (StringUtil.in(name, "br", "dd", "dt", "p", "h1", "h2", "h3", "h4", "h5"))
+ append("\n");
+ else if (name.equals("a"))
+ append(String.format(" <%s>", node.absUrl("href")));
+ }
+
+ // appends text to the string builder with a simple word wrap method
+ private void append(String text) {
+ if (text.startsWith("\n"))
+ width = 0; // reset counter if starts with a newline. only from formats above, not in natural text
+ if (text.equals(" ") &&
+ (accum.length() == 0 || StringUtil.in(accum.substring(accum.length() - 1), " ", "\n")))
+ return; // don't accumulate long runs of empty spaces
+
+ if (text.length() + width > maxWidth) { // won't fit, needs to wrap
+ String[] words = text.split("\\s+");
+ for (int i = 0; i < words.length; i++) {
+ String word = words[i];
+ boolean last = i == words.length - 1;
+ if (!last) // insert a space if not the last word
+ word = word + " ";
+ if (word.length() + width > maxWidth) { // wrap and reset counter
+ accum.append("\n").append(word);
+ width = word.length();
+ } else {
+ accum.append(word);
+ width += word.length();
+ }
+ }
+ } else { // fits as is, without need to wrap text
+ accum.append(text);
+ width += text.length();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return accum.toString();
+ }
+ }
+}