From cfaec454ef5fc40a00c13ba9f30e09db5208d5b7 Mon Sep 17 00:00:00 2001 From: chuanyu Date: Thu, 31 Aug 2017 09:02:06 -0700 Subject: [PATCH] #63 response to PR --- analysis/SConstruct | 3 -- analysis/config_global.yaml | 3 ++ analysis/release/sconstruct.log | 4 +- analysis/release/state_of_repo.log | 55 ++++++++++++++----------- config/config_python.py | 9 +++-- config/config_r.r | 13 +++--- config/configuration.py | 19 +++++++-- paper_slides/SConstruct | 26 ++++++------ paper_slides/config_global.yaml | 2 +- paper_slides/release/sconstruct.log | 4 +- paper_slides/release/state_of_repo.log | 56 +++++++++++++------------- 11 files changed, 111 insertions(+), 83 deletions(-) diff --git a/analysis/SConstruct b/analysis/SConstruct index f99bd97..c6d2d8b 100644 --- a/analysis/SConstruct +++ b/analysis/SConstruct @@ -19,13 +19,10 @@ log.start_log(mode, vers) env = Environment(ENV = {'PATH': os.environ['PATH']}, IMPLICIT_COMMAND_DEPENDENCIES = 0, BUILDERS = { - # 'BuildLatex' : Builder(action = gs.build_latex), - 'BuildLyx' : Builder(action = gs.build_lyx), # 'BuildMatlab': Builder(action = gs.build_matlab), 'BuildPython': Builder(action = gs.build_python), # 'BuildR' : Builder(action = gs.build_r), # 'BuildStata' : Builder(action = gs.build_stata), - 'Tablefill' : Builder(action = gs.build_tables) }) diff --git a/analysis/config_global.yaml b/analysis/config_global.yaml index 80143cd..787e042 100644 --- a/analysis/config_global.yaml +++ b/analysis/config_global.yaml @@ -7,3 +7,6 @@ build: source: prepare_data: source/prepare_data descriptive: source/descriptive + +# Switch to 'Yes' if it's required and 'No' if it's not. +prereq_git-lfs: Yes diff --git a/analysis/release/sconstruct.log b/analysis/release/sconstruct.log index 23b313b..688d801 100644 --- a/analysis/release/sconstruct.log +++ b/analysis/release/sconstruct.log @@ -1,9 +1,9 @@ -*** New build: {2017-08-30 16:06:12} *** +*** New build: {2017-08-31 09:01:34} *** scons: done reading SConscript files. scons: Building targets ... scons: `build' is up to date. scons: `release' is up to date. scons: done building targets. -*** Build completed: {2017-08-30 16:06:12} *** +*** Build completed: {2017-08-31 09:01:34} *** diff --git a/analysis/release/state_of_repo.log b/analysis/release/state_of_repo.log index 81cea82..29bfe40 100644 --- a/analysis/release/state_of_repo.log +++ b/analysis/release/state_of_repo.log @@ -8,19 +8,26 @@ as it can be edited after state_of_repo.log finishes running =================================== Last commit: -commit 465b8d9f106e4753bbe020d28d944c38128b1324 +commit 5d85886f5a4b83fbe91cb9988373921cdbbba603 Author: chuanyu -Date: Tue Aug 29 16:12:39 2017 -0700 +Date: Wed Aug 30 16:07:40 2017 -0700 - #63 add builder check + #63 add autimatic config scripts Files changed since last commit: -analysis/release/descriptive/plot.eps -analysis/release/descriptive/sconscript_descriptive.log +analysis/SConstruct +analysis/config_global.yaml analysis/release/sconstruct.log analysis/release/state_of_repo.log +config/config_python.py +config/config_r.r +config/configuration.py +paper_slides/SConstruct +paper_slides/config_global.yaml +paper_slides/release/sconstruct.log +paper_slides/release/state_of_repo.log =================================== @@ -28,49 +35,49 @@ analysis/release/state_of_repo.log =================================== ./config_global.yaml: - modified on: 30 Aug 2017 13:28:40 - size of file: 195 + modified on: 31 Aug 2017 08:51:04 + size of file: 273 ./config_user.yaml: modified on: 25 Aug 2017 11:06:38 size of file: 70 ./SConstruct: - modified on: 30 Aug 2017 16:02:14 - size of file: 2250 + modified on: 31 Aug 2017 08:50:15 + size of file: 2013 ./sconstruct.log: - modified on: 30 Aug 2017 16:06:12 + modified on: 31 Aug 2017 09:01:34 size of file: 199 ./state_of_repo.log: - modified on: 30 Aug 2017 16:06:12 - size of file: 576 + modified on: 31 Aug 2017 09:01:34 + size of file: 738 ./build/descriptive/plot.eps: - modified on: 30 Aug 2017 16:06:06 + modified on: 30 Aug 2017 16:56:33 size of file: 15385 ./build/descriptive/sconscript_descriptive.log: - modified on: 30 Aug 2017 16:06:06 + modified on: 30 Aug 2017 16:56:33 size of file: 99 ./build/descriptive/table.txt: - modified on: 30 Aug 2017 16:06:06 + modified on: 30 Aug 2017 16:56:33 size of file: 39 ./build/prepare_data/data.txt: - modified on: 28 Aug 2017 09:58:31 + modified on: 30 Aug 2017 16:56:32 size of file: 1988897 ./build/prepare_data/sconscript_create_data.log: - modified on: 28 Aug 2017 09:58:31 + modified on: 30 Aug 2017 16:56:32 size of file: 99 ./release/sconstruct.log: - modified on: 30 Aug 2017 16:06:06 - size of file: 692 + modified on: 31 Aug 2017 08:42:35 + size of file: 250 ./release/state_of_repo.log: - modified on: 30 Aug 2017 16:06:06 - size of file: 2737 + modified on: 31 Aug 2017 08:42:34 + size of file: 2744 ./release/descriptive/plot.eps: - modified on: 30 Aug 2017 16:06:06 + modified on: 31 Aug 2017 08:17:47 size of file: 15385 ./release/descriptive/sconscript_descriptive.log: - modified on: 30 Aug 2017 16:06:06 + modified on: 31 Aug 2017 08:17:47 size of file: 99 ./release/descriptive/table.txt: - modified on: 30 Aug 2017 16:06:06 + modified on: 30 Aug 2017 16:56:33 size of file: 39 ./release/lg/data.txt: modified on: 28 Aug 2017 09:58:31 diff --git a/config/config_python.py b/config/config_python.py index 6538b77..61f9322 100644 --- a/config/config_python.py +++ b/config/config_python.py @@ -1,13 +1,16 @@ import pip import pkg_resources +# The collection of required packages. +# Packages with value True will be updated if it's already installed. +# Packages with value False will be skipped if it's already installed. +packages = {'pyyaml': True} -def main(): +def main(packages): # Install gslab_tools if it's not installed yet or it has the wrong version if not check_gslab_tools('4.1.0'): pip.main(['install', 'git+http://git@github.com/gslab-econ/gslab_python.git@4.1.0']) # Install required packages using pip - packages = {'pyyaml': True} installed_packages = [pkg.key for pkg in pip.get_installed_distributions()] for pkg in packages.keys(): if pkg not in installed_packages or packages[pkg]: @@ -31,4 +34,4 @@ def check_gslab_tools(gslab_python_version): else: return True -main() +main(packages) diff --git a/config/config_r.r b/config/config_r.r index 3f72784..98a4ee5 100644 --- a/config/config_r.r +++ b/config/config_r.r @@ -1,7 +1,10 @@ -Main <- function(){ - CRAN_packages <- c("yaml", "ggplot2") - GitHub_packages <- NULL - # GitHub packages should be the full path to the relevant repo +# Required packages installed from CRAN. +CRAN_packages <- c("yaml") +# Required packages installed from Github. Should be the full path to the relevant repo. +GitHub_packages <- NULL + +main <- function(CRAN_packages = NULL, GitHub_packages = NULL) { + # If there are packages installed from Github, first make sure "devtools" is installed if (!is.null(GitHub_packages)) { CRAN_packages <- c(CRAN_packages, "devtools") } @@ -25,4 +28,4 @@ install_CRAN <- function(pkg, repo, dependency, quiet){ } } -Main() +main(CRAN_packages, GitHub_packages) diff --git a/config/configuration.py b/config/configuration.py index 8d134f9..60afe4c 100644 --- a/config/configuration.py +++ b/config/configuration.py @@ -5,7 +5,8 @@ from gslab_scons import misc from gslab_scons import configuration_tests as config -def configuration(ARGUMENTS, config_user_yaml = 'config_user.yaml'): +def configuration(ARGUMENTS, paper = False, config_user_yaml = 'config_user.yaml', + config_global_yaml = 'config_global.yaml'): # Determines whether to print traceback messages debug = ARGUMENTS.get('debug', False) if not debug: @@ -13,9 +14,19 @@ def configuration(ARGUMENTS, config_user_yaml = 'config_user.yaml'): # http://stackoverflow.com/questions/27674602/hide-traceback-unless-a-debug-flag-is-set sys.tracebacklimit = 0 - # Checks git-lfs and lyx - config.check_lfs() - config.check_lyx() + # Checks git-lfs + prereq_gitlfs = misc.load_yaml_value(config_global_yaml, 'prereq_git-lfs') + if prereq_gitlfs: + config.check_lfs() + + # Check lyx or latex + if paper: + prereq_lyx = misc.load_yaml_value(config_global_yaml, 'prereq_Lyx') + prereq_latex = misc.load_yaml_value(config_global_yaml, 'prereq_Latex') + if prereq_lyx: + config.check_lyx() + if prereq_latex: + pass # Loads arguments and configurations mode = ARGUMENTS.get('mode', 'develop') # Gets mode; defaults to 'develop' diff --git a/paper_slides/SConstruct b/paper_slides/SConstruct index 02c0b52..e1cf9a1 100644 --- a/paper_slides/SConstruct +++ b/paper_slides/SConstruct @@ -1,27 +1,29 @@ # Preliminaries import os import sys +import atexit +import yaml +sys.path.append('../config') sys.dont_write_bytecode = True # Don't write .pyc files -# Test for proper prerequisites and setup -from configuration_test import configuration_test -[mode, cache_dir] = configuration_test(ARGUMENTS, gslab_python_version = '4.1.0') +# Setup +from configuration import configuration +[mode, vers, cache_dir] = configuration(ARGUMENTS, paper = True) import gslab_scons as gs -import gslab_scons.log as log -import yaml -import atexit +from gslab_scons import log +from gslab_scons import misc -# Start log after getting mode and release version -mode = ARGUMENTS.get('mode', 'develop') -vers = ARGUMENTS.get('version', '') +# Start log log.start_log(mode, vers) # Define the SCons environment env = Environment(ENV = {'PATH': os.environ['PATH']}, IMPLICIT_COMMAND_DEPENDENCIES = 0, - BUILDERS = {'BuildLyx': Builder(action = gs.build_lyx), - 'BuildLatex': Builder(action = gs.build_latex), - 'Tablefill': Builder(action = gs.build_tables)}) + BUILDERS = { + 'BuildLyx': Builder(action = gs.build_lyx), + # 'BuildLatex': Builder(action = gs.build_latex), + 'Tablefill': Builder(action = gs.build_tables), + }) # Only computes hash if time-stamp changed env.Decider('MD5-timestamp') diff --git a/paper_slides/config_global.yaml b/paper_slides/config_global.yaml index c7394b8..e09bcc8 100644 --- a/paper_slides/config_global.yaml +++ b/paper_slides/config_global.yaml @@ -14,7 +14,7 @@ source: tables: source/tables talk: source/talk -# Switch to 'Yes' if any of the following softwares are required +# Switch to 'Yes' if it's required and 'No' if it's not. prereq_git-lfs: Yes prereq_Latex: No prereq_Lyx: Yes diff --git a/paper_slides/release/sconstruct.log b/paper_slides/release/sconstruct.log index b38c6b1..2f9052a 100644 --- a/paper_slides/release/sconstruct.log +++ b/paper_slides/release/sconstruct.log @@ -1,9 +1,9 @@ -*** New build: {2017-08-25 16:32:48} *** +*** New build: {2017-08-31 09:01:29} *** scons: done reading SConscript files. scons: Building targets ... scons: `build' is up to date. scons: `release' is up to date. scons: done building targets. -*** Build completed: {2017-08-25 16:32:49} *** +*** Build completed: {2017-08-31 09:01:30} *** diff --git a/paper_slides/release/state_of_repo.log b/paper_slides/release/state_of_repo.log index 8e9f05d..d259e43 100644 --- a/paper_slides/release/state_of_repo.log +++ b/paper_slides/release/state_of_repo.log @@ -8,22 +8,24 @@ as it can be edited after state_of_repo.log finishes running =================================== Last commit: -commit 5fafb61cb4248d84dd6a94b523b948768c8d2dda +commit 5d85886f5a4b83fbe91cb9988373921cdbbba603 Author: chuanyu -Date: Thu Aug 24 17:02:41 2017 -0700 +Date: Wed Aug 30 16:07:40 2017 -0700 - #64 turn on git-lfs + #63 add autimatic config scripts Files changed since last commit: analysis/SConstruct analysis/config_global.yaml -analysis/configuration_test.py analysis/release/sconstruct.log analysis/release/state_of_repo.log +config/config_python.py +config/config_r.r +config/configuration.py +paper_slides/SConstruct paper_slides/config_global.yaml -paper_slides/configuration_test.py paper_slides/release/sconstruct.log paper_slides/release/state_of_repo.log @@ -33,20 +35,20 @@ paper_slides/release/state_of_repo.log =================================== ./config_global.yaml: - modified on: 25 Aug 2017 16:32:20 - size of file: 365 + modified on: 31 Aug 2017 09:00:44 + size of file: 357 ./configuration_test.py: - modified on: 25 Aug 2017 16:32:33 + modified on: 28 Aug 2017 09:19:50 size of file: 2016 ./SConstruct: - modified on: 25 Aug 2017 16:05:28 - size of file: 1877 + modified on: 31 Aug 2017 09:01:21 + size of file: 1820 ./sconstruct.log: - modified on: 25 Aug 2017 16:32:49 + modified on: 31 Aug 2017 09:01:30 size of file: 199 ./state_of_repo.log: - modified on: 25 Aug 2017 16:32:49 - size of file: 701 + modified on: 31 Aug 2017 09:01:30 + size of file: 738 ./build/paper/ondeck.pdf: modified on: 24 Aug 2017 16:51:29 size of file: 18193 @@ -93,7 +95,7 @@ paper_slides/release/state_of_repo.log modified on: 24 Aug 2017 16:52:19 size of file: 29971 ./input/plot.eps: - modified on: 25 Aug 2017 16:05:28 + modified on: 28 Aug 2017 09:19:50 size of file: 15385 ./input/provenance.txt: modified on: 23 Aug 2017 12:00:48 @@ -102,31 +104,31 @@ paper_slides/release/state_of_repo.log modified on: 24 Aug 2017 15:33:21 size of file: 39 ./release/sconstruct.log: - modified on: 25 Aug 2017 16:32:38 + modified on: 31 Aug 2017 08:59:45 size of file: 250 ./release/state_of_repo.log: - modified on: 25 Aug 2017 16:32:38 - size of file: 4478 + modified on: 31 Aug 2017 08:59:45 + size of file: 4589 ./release/paper/ondeck.pdf: - modified on: 25 Aug 2017 16:05:28 + modified on: 28 Aug 2017 09:19:50 size of file: 18193 ./release/paper/online_appendix.pdf: - modified on: 25 Aug 2017 16:05:28 + modified on: 28 Aug 2017 09:19:50 size of file: 45354 ./release/paper/paper.pdf: - modified on: 25 Aug 2017 16:05:28 + modified on: 28 Aug 2017 09:19:50 size of file: 58961 ./release/paper/sconscript.log: - modified on: 25 Aug 2017 16:05:28 + modified on: 28 Aug 2017 09:19:50 size of file: 6473 ./release/paper/text.pdf: - modified on: 25 Aug 2017 16:05:28 + modified on: 28 Aug 2017 09:19:50 size of file: 18193 ./release/talk/sconscript.log: - modified on: 25 Aug 2017 16:05:28 + modified on: 28 Aug 2017 09:19:50 size of file: 589 ./release/talk/slides.pdf: - modified on: 25 Aug 2017 16:05:28 + modified on: 28 Aug 2017 09:19:50 size of file: 29971 ./source/figures/figure.lyx: modified on: 23 Aug 2017 12:00:56 @@ -141,13 +143,13 @@ paper_slides/release/state_of_repo.log modified on: 23 Aug 2017 12:00:56 size of file: 5236 ./source/paper/SConscript: - modified on: 25 Aug 2017 16:05:28 - size of file: 676 + modified on: 28 Aug 2017 09:19:50 + size of file: 677 ./source/paper/text.lyx: modified on: 23 Aug 2017 12:00:56 size of file: 2522 ./source/paper/text.tex: - modified on: 25 Aug 2017 16:05:28 + modified on: 28 Aug 2017 09:19:50 size of file: 1300 ./source/tables/SConscript: modified on: 23 Aug 2017 12:00:56