Pull request for #73: Allows tablefill to use tex documents (#82)

* #73 Preliminary implementation

* #73 Updates info

* #73 add newline after log for easy viewing

gslab_fill/tablefill.py

@@ -14,7 +14,6 @@ def tablefill(**kwargs):
         args = parse_arguments(kwargs)
         tables = parse_tables(args)
         lyx_text = insert_tables(args, tables)
-        lyx_text = insert_warning(args, lyx_text)
         write_to_lyx(args, lyx_text)
         exitmessage = args['template'] + ' filled successfully by tablefill'
         print exitmessage
@@ -106,7 +105,7 @@ def insert_tables(args,tables):
                         lyx_text[i] = lyx_text[i].replace('#' + entry_tag + '#', rounded_entry)
                         entry_count+=1
 
-                    elif lyx_text[i]=='</lyxtabular>\n':
+                    elif lyx_text[i] == '</lyxtabular>\n' or lyx_text[i] == "end{tabular}":
                         search_table = False
 
     return lyx_text
@@ -140,28 +139,6 @@ def insert_commas(entry):
     return entry_commas
 
 
-def insert_warning(args, lyx_text):
-    input = ' '.join(args['input'])
-    template = ''.join(args['template'])
-    message = '\n\\begin_layout Standard\n\\begin_inset Note Note\nstatus open' \
-               '\n\n\\begin_layout Plain Layout\nThis file was produced by ' \
-               'tablefill.py from template file %s and input file(s) %s.  To make '\
-               'changes in this file, edit the input and template files. Do not '\
-               'edit this file directly.\n\\end_layout\n\n\\end_inset\n\n\\end_layout\n'
-    filled_message = message % (template, input)
-    message_lines = filled_message.split('\n')
-    message_lines = [s + '\n' for s in message_lines]
-    printed = False
-    n = -1
-    while not printed:
-        n += 1
-        if lyx_text[n].startswith('\\begin_body'):
-            lyx_text[n+1:n+1] = message_lines
-            printed = True
-
-    return lyx_text
-
-
 def write_to_lyx(args, lyx_text):    
     outfile = open(args['output'], 'wb')
     outfile.write( ''.join(lyx_text) )

gslab_fill/tablefill_info.py

@@ -4,13 +4,14 @@
 #################################################################
 
 Description:
-tablefill.py is a Python module designed to fill LyX tables with output 
+tablefill.py is a Python module designed to fill LyX/Tex tables with output 
 from text files (usually output from Stata or Matlab).
 
 Usage:
-Tablefill takes as input a LyX file containing empty tables (the template 
+Tablefill takes as input a LyX (or Tex) file containing empty tables (the template 
 file) and text files containing data to be copied to  these tables (the 
-input  files), and produces a LyX file with filled tables (the output file). 
+input  files), and produces a LyX (or Tex) file with filled tables (the output file). 
+For brevity, LyX will be used to denote LyX or Tex files throughout.
 
 Tablefill must first be imported to make.py.  This is typically achieved 
 by including the following lines:

gslab_fill/tests/test_tablefill.py

@@ -31,9 +31,9 @@ def testInput(self):
         self.assertIn('filled successfully', message)                        
         tag_data = open('../../gslab_fill/tests/input/tablefill_template.lyx', 'rU').readlines()
         filled_data = open('./build/tablefill_template_filled.lyx', 'rU').readlines()
-        self.assertEqual(len(tag_data) + 13, len(filled_data))
+        self.assertEqual(len(tag_data), len(filled_data))
         for n in range(len(tag_data)):
-            self.tag_compare(tag_data[n], filled_data[n + 13]) 
+            self.tag_compare(tag_data[n], filled_data[n]) 
 
     def tag_compare(self, tag_line, filled_line):
         if re.match('^.*#\d+#', tag_line) or re.match('^.*#\d+,#', tag_line):
@@ -65,7 +65,7 @@ def testIllegalSyntax(self):
         with nostderrout():
 
             error = tablefill(input   = '../../gslab_fill/tests/input/tables_appendix.txt ' + \
-                                       ' ../../gslab_fill/tests/input/tables_appendix_two.txt', 
+                                        '../../gslab_fill/tests/input/tables_appendix_two.txt', 
                               template = '../../gslab_fill/tests/input/textfill_template.lyx')
         self.assertIn('KeyError', error)
 

gslab_scons/builders/build_tables.py

@@ -7,7 +7,7 @@ def build_tables(target, source, env):
     '''Build a SCons target by filling a table
 
     This function uses the tablefill function from gslab_fill to produced a 
-    filled table from (i) an empty table in a LyX file and (ii) text files 
+    filled table from (i) an empty table in a LyX/Tex file and (ii) text files 
     containing data to be used in filling the table. 
 
     Parameters
@@ -16,7 +16,7 @@ def build_tables(target, source, env):
         The target(s) of the SCons command.
     source: string or list
         The source(s) of the SCons command. The first source specified
-        should be the LyX file specifying the table format. The subsequent 
+        should be the LyX/Tex file specifying the table format. The subsequent 
         sources should be the text files containing the data with which the
         tables are to be filled. 
     env: SCons construction environment, see SCons user guide 7.2
@@ -29,30 +29,31 @@ def build_tables(target, source, env):
 
     # Set up source file (table format)
     source_file = str(source[0])
-    misc.check_code_extension(source_file, '.lyx')
+    misc.check_code_extension(source_file, ['.lyx', '.tex'])
 
     # Set up input string (list of data tables)
     input_string = ' '.join([str(i) for i in source[1:]])
 
     # Set up target file (filled table)
     target_file = str(target[0])
     target_dir  = misc.get_directory(target_file)    
-    misc.check_code_extension(target_file, '.lyx')
+    misc.check_code_extension(target_file, ['.lyx', '.tex'])
     log_file = target_dir + '/sconscript.log'
 
     # Command call
-    command = """tablefill(input    = %s, 
-                         template = %s, 
-                         output   = %s)""" % (input_string, source_file, target_file)
     output  = tablefill(input    = input_string, 
                         template = source_file, 
                         output   = target_file)
 
-    # Close log
     with open(log_file, 'wb') as f:
         f.write(output)
-
-    if "traceback" in str.lower(output): # if tablefill.py returns an error            
+        f.write("\n")
+
+    # Close log
+    if "traceback" in str.lower(output): # if tablefill.py returns an error   
+        command = """tablefill(input    = %s, 
+                         template = %s, 
+                         output   = %s)""" % (input_string, source_file, target_file)         
         message = misc.command_error_msg("tablefill.py", command)
         raise ExecCallError(message)
 

gslab_scons/misc.py

@@ -197,14 +197,22 @@ def make_list_if_string(source):
     return source
 
 
-def check_code_extension(source_file, extension):
+def check_code_extension(source_file, extensions):
     '''
     This function raises an exception if the extension in `source_file`
     does not match the software package specified by `software`.
     '''
+    if not isinstance(extensions, list):
+        extensions = [extensions]
     source_file = str.lower(str(source_file))
-    extension   = str.lower(str(extension))
-    if not source_file.endswith(extension):
+    error       = True
+    for extension in extensions:
+        extension = str.lower(str(extension))
+
+        if source_file.endswith(extension):
+            error = False
+
+    if error:
         error_message = 'First argument, %s, must be a %s file.' % \
                 (source_file, extension)
         raise _exception_classes.BadExtensionError(error_message)

gslab_scons/tests/test_build_lyx.py

@@ -69,7 +69,7 @@ def test_env_argument(self, mock_system):
         log    = './build/sconscript.log'
 
         for env in [True, [1, 2, 3], ('a', 'b'), None, TypeError]:
-            with self.assertRaises(ExecCallError):
+            with self.assertRaises(ExecCallError, TypeError):
                 gs.build_lyx(target, source, env = env)
 
     @mock.patch('%s.os.system' % path)

gslab_scons/tests/test_build_tables.py

@@ -11,18 +11,30 @@
 sys.path.append('../..')
 
 import gslab_scons.builders.build_tables as gs
-from gslab_scons._exception_classes import BadExtensionError
+from gslab_scons._exception_classes import BadExtensionError, ExecCallError
 from gslab_make.tests import nostderrout
 
 
 class TestBuildTables(unittest.TestCase):
 
+    def setUp(self):
+        if not os.path.exists('./build/'):
+            os.mkdir('./build/')
+
+    def table_fill_side_effect(self, input, template, output):
+        return ""
+
+    def table_fill_side_effect_error(self, input, template, output):
+        return "traceback"
+
     @mock.patch('gslab_scons.builders.build_tables.tablefill')
     def test_standard(self, mock_tablefill):
         '''
         Test that build_tables() correctly prepares and passes
         inputs to the gslab_fill.tablefill() function
         '''
+
+        mock_tablefill.side_effect = self.table_fill_side_effect
         # Specify the sources and the target arguments of build_tables()
         source = ['./input/tablefill_template.lyx', 
                   './input/tables_appendix.txt',
@@ -38,6 +50,11 @@ def test_standard(self, mock_tablefill):
         gs.build_tables(target, source, '')
         self.check_call(source, target, mock_tablefill)
 
+        # The target can also be a tex file
+        target = ('./build/tablefill_template_filled.tex')
+        gs.build_tables(target, source, '')
+        self.check_call(source, target, mock_tablefill)
+
     def check_call(self, source, target, mock_tablefill):
         '''
         This method checks that the build_tables() behaves as expected
@@ -71,6 +88,7 @@ def test_default_string_target(self, mock_tablefill):
         Test that build_tables() constructs LyX tables correctly when
         its target argument is a string.
         '''
+        mock_tablefill.side_effect = self.table_fill_side_effect
         source = ['./input/tablefill_template.lyx', 
                   './input/tables_appendix.txt', 
                   './input/tables_appendix_two.txt']
@@ -79,6 +97,20 @@ def test_default_string_target(self, mock_tablefill):
 
         self.check_call(source, target, mock_tablefill)      
 
+    @mock.patch('gslab_scons.builders.build_tables.tablefill')
+    def test_error_traceback(self, mock_tablefill):
+        '''
+        Test that build_tables() properly outputs traceback.
+        '''
+        mock_tablefill.side_effect = self.table_fill_side_effect_error
+        source = ['./input/tablefill_template.lyx', 
+                  './input/tables_appendix.txt', 
+                  './input/tables_appendix_two.txt']
+        target = './build/tablefill_template_filled.lyx'
+        with self.assertRaises(ExecCallError), nostderrout():
+            gs.build_tables(target, source, '')
+
+
     def test_target_extension(self):
         '''Test that build_tables() recognises an inappropriate file extension'''
 
@@ -103,6 +135,8 @@ def test_unintended_inputs(self, mock_tablefill):
                       './input/tables_appendix.txt']
         std_target = './build/tablefill_template_filled.lyx'
 
+        mock_tablefill.side_effect = self.table_fill_side_effect
+
         #== env =============
         # We expect that build_tables() will accept any env argument
         for env in ['', None, Exception, "the environment", (1, 2, 3)]:
@@ -133,7 +167,11 @@ def test_unintended_inputs(self, mock_tablefill):
         # source is a non-strings non-iterable with no len() value
         source = 1
         with self.assertRaises(TypeError):
-            gs.build_tables(std_target, source, None)
+            gs.build_tables(std_target, source, None)\
+
+    def tearDown(self):
+        if os.path.exists('./build/'):
+            shutil.rmtree('./build/')
 
 
 if __name__ == '__main__':

test.log

@@ -1,9 +1,4 @@
 running pytest
-Searching for pytest
-Best match: pytest 3.0.6
-Processing pytest-3.0.6-py2.7.egg
-
-Using /Users/mrsull/Desktop/gslab_python/.eggs/pytest-3.0.6-py2.7.egg
 running egg_info
 writing requirements to GSLab_Tools.egg-info/requires.txt
 writing GSLab_Tools.egg-info/PKG-INFO
@@ -13,9 +8,9 @@ reading manifest file 'GSLab_Tools.egg-info/SOURCES.txt'
 writing manifest file 'GSLab_Tools.egg-info/SOURCES.txt'
 running build_ext
 ============================= test session starts ==============================
-platform darwin -- Python 2.7.12, pytest-3.0.6, py-1.4.31, pluggy-0.4.0
-rootdir: /Users/mrsull/Desktop/gslab_python, inifile: 
-collected 122 items
+platform darwin -- Python 2.7.13, pytest-3.0.5, py-1.4.32, pluggy-0.4.0
+rootdir: /Users/leviboxell/Desktop/gslab_python, inifile: 
+collected 123 items
 
 gencat/tests/test_checkDicts.py ...
 gencat/tests/test_cleanDir.py ......
@@ -30,15 +25,15 @@ gslab_scons/tests/test_build_matlab.py ......
 gslab_scons/tests/test_build_python.py ......
 gslab_scons/tests/test_build_r.py .....
 gslab_scons/tests/test_build_stata.py ..........
-gslab_scons/tests/test_build_tables.py ....
-gslab_scons/tests/test_configuration_tests.py ...........
+gslab_scons/tests/test_build_tables.py .....
+gslab_scons/tests/test_configuration_tests.py .........
 gslab_scons/tests/test_log.py .........
-gslab_scons/tests/test_misc.py ................
+gslab_scons/tests/test_misc.py ..................
 gslab_scons/tests/test_release_function.py ....
 gslab_scons/tests/test_release_tools.py ....
 gslab_scons/tests/test_size_warning.py .....
 
-========================== 122 passed in 2.07 seconds ==========================
+========================== 123 passed in 1.75 seconds ==========================
 Name                                            Stmts   Miss Branch BrPart  Cover   Missing
 -------------------------------------------------------------------------------------------
 gencat/gencat.py                                   79      6     28      4    91%   72-73, 81, 89, 97, 101, 57->exit, 71->72, 96->97, 100->101
@@ -49,7 +44,7 @@ gencat/tests/test_main.py                          49      4      8      1    91
 gencat/tests/test_unzipFiles.py                   124      6     30      2    95%   15, 17, 30-32, 194, 156->150, 193->194
 gencat/tests/test_writeDict.py                     60      6      8      1    90%   15, 17, 30-32, 92, 91->92
 gencat/tests/test_zipFile.py                      101      6      8      1    94%   16, 19, 28-30, 149, 148->149
-gslab_fill/tablefill.py                           123      4     48      7    94%   54, 101, 135, 138, 33->36, 36->38, 53->54, 86->83, 100->101, 131->135, 137->138
+gslab_fill/tablefill.py                           107      4     44      7    93%   53, 100, 134, 137, 32->35, 35->37, 52->53, 85->82, 99->100, 130->134, 136->137
 gslab_fill/tablefill_info.py                        0      0      0      0   100%
 gslab_fill/tests/test_tablefill.py                 70      2     16      4    93%   118-119, 22->exit, 52->exit, 113->exit, 117->118
 gslab_fill/tests/test_textfill.py                  84      2     12      3    95%   139-140, 18->exit, 134->exit, 138->139
@@ -71,18 +66,18 @@ gslab_make/private/preliminaries.py               107     88     40      0    13
 gslab_make/private/runprogramdirective.py         207    185    104      0     7%   16-93, 97-107, 111-116, 121-134, 138-145, 149-163, 167-220, 224-240, 244-264, 271-282, 285-289, 296-302
 gslab_make/run_program.py                         238    217     80      0     7%   129-162, 178-202, 208-225, 231-248, 254-275, 281-298, 304-321, 332-392, 398-423, 429-447, 453-488, 494-499
 gslab_make/tests/nostderrout.py                    12      0      2      0   100%
-gslab_scons/_exception_classes.py                  10      0      0      0   100%
-gslab_scons/_release_tools.py                     128      0     48      0   100%
+gslab_scons/_exception_classes.py                  11      0      0      0   100%
+gslab_scons/_release_tools.py                     132      0     46      1    99%   34->37
 gslab_scons/builders/build_lyx.py                  28      0      0      0   100%
 gslab_scons/builders/build_matlab.py               34      0      4      0   100%
 gslab_scons/builders/build_python.py               24      0      0      0   100%
-gslab_scons/builders/build_r.py                    26      0      2      0   100%
+gslab_scons/builders/build_r.py                    28      1      4      1    94%   41, 38->41
 gslab_scons/builders/build_stata.py                30      0      0      0   100%
-gslab_scons/builders/build_tables.py               11      0      0      0   100%
-gslab_scons/configuration_tests.py                133      0     48      0   100%
-gslab_scons/log.py                                 53      8     16      2    86%   53-54, 79-84, 52->53, 78->79
-gslab_scons/misc.py                               132      0     48      0   100%
-gslab_scons/release.py                             38     38     18      0     0%   1-65
+gslab_scons/builders/build_tables.py               25      0      2      0   100%
+gslab_scons/configuration_tests.py                 94      0     34      0   100%
+gslab_scons/log.py                                 64     18     18      2    73%   52-56, 81-93, 51->52, 80->81
+gslab_scons/misc.py                               184      3     72      3    98%   257-258, 289, 58->46, 256->257, 288->289
+gslab_scons/release.py                             41     41     20      0     0%   1-72
 gslab_scons/size_warning.py                        69      0     30      0   100%
 gslab_scons/tests/_side_effects.py                112      2     28      7    94%   34-35, 31->34, 37->exit, 49->exit, 73->78, 144->156, 160->exit, 246->252
 gslab_scons/tests/_test_helpers.py                 95      4     34      3    93%   27-28, 102, 117, 76->88, 99->102, 112->117
@@ -91,12 +86,12 @@ gslab_scons/tests/test_build_matlab.py             54      1      6      1    97
 gslab_scons/tests/test_build_python.py             60      2     14      4    92%   75, 107, 23->exit, 74->75, 102->exit, 106->107
 gslab_scons/tests/test_build_r.py                  46      1     12      3    93%   88, 27->exit, 83->exit, 87->88
 gslab_scons/tests/test_build_stata.py              90      1      2      1    98%   179, 178->179
-gslab_scons/tests/test_build_tables.py             65      1      8      1    97%   140, 139->140
-gslab_scons/tests/test_configuration_tests.py     253      3     72      9    96%   212, 244, 401, 108->exit, 207->214, 209->212, 243->244, 283->exit, 290->exit, 315->exit, 343->exit, 400->401
+gslab_scons/tests/test_build_tables.py             87      1     12      3    96%   178, 21->exit, 173->exit, 177->178
+gslab_scons/tests/test_configuration_tests.py     194      3     54      5    97%   211, 226, 313, 107->exit, 206->213, 208->211, 225->226, 312->313
 gslab_scons/tests/test_log.py                     112     10     12      3    86%   27, 152-158, 194, 198, 26->27, 193->194, 197->198
-gslab_scons/tests/test_misc.py                    136      2     10      2    97%   177, 275, 176->177, 274->275
-gslab_scons/tests/test_release_function.py        122      3     22      2    97%   288-289, 322, 182->exit, 321->322
+gslab_scons/tests/test_misc.py                    192      2     28      6    96%   179, 363, 178->179, 282->exit, 289->exit, 314->exit, 342->exit, 362->363
+gslab_scons/tests/test_release_function.py        121      3     22      2    97%   293-294, 324, 184->exit, 323->324
 gslab_scons/tests/test_release_tools.py            64      1      2      1    97%   153, 152->153
 gslab_scons/tests/test_size_warning.py            183      5     46      6    95%   276, 293, 309, 373, 379, 272->276, 273->276, 292->293, 304->309, 370->373, 378->379
 -------------------------------------------------------------------------------------------
-TOTAL                                            4515   1284   1278     78    66%
+TOTAL                                            4565   1301   1294     85    66%