Trac #20182: Automatic doctest for external softwares

Many doctests depend on softwares external to Sage (including
"internet") available on the system, such as latex, magma, mathematica,
maple, etc. This patch allows to run the optional doctests for the
softwares available.

With the patch,

"sage -t --optional=PKGS": only run doctests including one of the "#
optional - xxx" tags where xxx is listed in         PKGS;
    if "sage" is listed, will also run the standard doctests;
    if "optional" is listed, will also run tests for installed optional
(new-style) packages;
    if "external" is listed, will also run tests for available external
softwares;
    if set to "all", then all tests will be run

This is a rewrite of #18904 and #13540.

URL: http://trac.sagemath.org/20182
Reported by: klee
Ticket author(s): Kwankyu Lee
Reviewer(s): John Palmieri

Makefile

@@ -91,34 +91,37 @@ test: all
 check: test
 
 testall: all
-	$(TESTALL) --optional=all --logfile=logs/testall.log
+	$(TESTALL) --optional=sage,optional,external --logfile=logs/testall.log
 
 testlong: all
 	$(TESTALL) --long --logfile=logs/testlong.log
 
 testalllong: all
-	$(TESTALL) --long --optional=all --logfile=logs/testalllong.log
+	$(TESTALL) --long --optional=sage,optional,external --logfile=logs/testalllong.log
 
 ptest: all
 	$(PTESTALL) --logfile=logs/ptest.log
 
 ptestall: all
-	$(PTESTALL) --optional=all --logfile=logs/ptestall.log
+	$(PTESTALL) --optional=sage,optional,external --logfile=logs/ptestall.log
 
 ptestlong: all
 	$(PTESTALL) --long --logfile=logs/ptestlong.log
 
 ptestalllong: all
-	$(PTESTALL) --long --optional=all --logfile=logs/ptestalllong.log
+	$(PTESTALL) --long --optional=sage,optional,external --logfile=logs/ptestalllong.log
 
+testoptional: all
+	$(TESTALL) --optional=sage,optional --logfile=logs/testoptional.log
 
-testoptional: testall # just an alias
+testoptionallong: all
+	$(TESTALL) --long --optional=sage,optional --logfile=logs/testoptionallong.log
 
-testoptionallong: testalllong # just an alias
+ptestoptional: all
+	$(PTESTALL) --optional=sage,optional --logfile=logs/ptestoptional.log
 
-ptestoptional: ptestall # just an alias
-
-ptestoptionallong: ptestalllong # just an alias
+ptestoptionallong: all
+	$(PTESTALL) --long --optional=sage,optional --logfile=logs/ptestoptionallong.log
 
 configure: configure.ac src/bin/sage-version.sh m4/*.m4
 	./bootstrap -d

src/bin/sage-runtests

@@ -29,9 +29,10 @@ if __name__ == "__main__":
         metavar="SECONDS", help="warn if tests take more time than SECONDS")
     parser.add_option("--optional", metavar="PKGS", default="sage,optional",
         help='only run tests including one of the "# optional" tags listed in PKGS; '
-             'if "sage" is listed will also test the standard doctests; '
-             'if "optional" is listed will also test all available optional (new-style) packages; '
-             'if set to "all", then all tests will be run; ')
+             'if "sage" is listed, will also run the standard doctests; '
+             'if "optional" is listed, will also run tests for installed optional (new-style) packages; '
+             'if "external" is listed, will also run tests for available external software; '
+             'if set to "all", then all tests will be run')
     parser.add_option("--randorder", type=int, metavar="SEED", help="randomize order of tests")
     parser.add_option("--global-iterations", "--global_iterations", type=int, default=0, help="repeat the whole testing process this many times")
     parser.add_option("--file-iterations", "--file_iterations", type=int, default=0, help="repeat each file this many times, stopping on the first failure")

src/doc/en/developer/doctesting.rst

@@ -830,6 +830,25 @@ In order to just run the tests that are marked as requiring magma, omit ``sage``
         cpu time: 0.1 seconds
         cumulative wall time: 2.0 seconds
 
+If you want Sage to detect external software or other capabilities
+(such as magma, latex, internet) automatically and run all of the
+relevant tests, then add ``external``::
+
+    $ sage -t --optional=external src/sage/rings/real_mpfr.pyx 
+    Running doctests with ID 2016-03-16-14-10-21-af2ebb67.
+    Using --optional=external
+    External software to be detected: cplex,gurobi,internet,latex,macaulay2,magma,maple,mathematica,matlab,octave,scilab
+    Doctesting 1 file.
+    sage -t --warn-long 28.0 src/sage/rings/real_mpfr.pyx
+        [5 tests, 0.04 s]
+    ----------------------------------------------------------------------
+    All tests passed!
+    ----------------------------------------------------------------------
+    Total time for all tests: 0.5 seconds
+        cpu time: 0.0 seconds
+        cumulative wall time: 0.0 seconds
+    External software detected for doctesting: magma
+
 To run all tests, regardless of whether they are marked optional, pass ``all`` as the ``optional`` tag::
 
     [roed@sage sage-6.0]$ sage -t --optional=all src/sage/rings/real_mpfr.pyx

src/doc/en/reference/doctest/index.rst

@@ -9,6 +9,7 @@ Sage's Doctesting Framework
    sage/doctest/forker
    sage/doctest/parsing
    sage/doctest/reporting
+   sage/doctest/external
    sage/doctest/test
    sage/doctest/util
    sage/doctest/fixtures

src/sage/doctest/control.py

@@ -31,6 +31,7 @@
 from forker import DocTestDispatcher
 from reporting import DocTestReporter
 from util import NestedName, Timer, count_noun, dict_difference
+from external import external_software, available_software
 
 nodoctest_regex = re.compile(r'\s*(#+|%+|r"+|"+|\.\.)\s*nodoctest')
 optionaltag_regex = re.compile(r'^\w+$')
@@ -78,7 +79,7 @@ def __init__(self, **kwds):
         self.sagenb = False
         self.long = False
         self.warn_long = None
-        self.optional = set(["sage"])
+        self.optional = set(['sage'])
         self.randorder = None
         self.global_iterations = 1  # sage-runtests default is 0
         self.file_iterations = 1    # sage-runtests default is 0
@@ -596,7 +597,7 @@ def expand_files_into_sources(self):
             sage: DC = DocTestController(DD, [dirname])
             sage: DC.expand_files_into_sources()
             sage: len(DC.sources)
-            10
+            11
             sage: DC.sources[0].options.optional
             True
 
@@ -697,6 +698,7 @@ def sort_sources(self):
             sage.doctest.parsing
             sage.doctest.forker
             sage.doctest.fixtures
+            sage.doctest.external
             sage.doctest.control
             sage.doctest.all
             sage.doctest
@@ -1032,12 +1034,18 @@ def run(self):
                     pass
 
             self.log("Using --optional=" + self._optional_tags_string())
+            if self.options.optional is True or 'external' in self.options.optional:
+                self.log("External software to be detected: " + ','.join(external_software))
 
             self.add_files()
             self.expand_files_into_sources()
             self.filter_sources()
             self.sort_sources()
             self.run_doctests()
+
+            if self.options.optional is True or 'external' in self.options.optional:
+                self.log("External software detected for doctesting: "
+                         + ','.join(available_software.seen()))
             return self.reporter.error_status
 
 def run_doctests(module, options=None):