Merge pull request #1047 from slacgismo/develop

Release beauharnois-30

.gitignore

@@ -178,3 +178,8 @@ gldcore/gridlabd.in
 geodata.conf
 gldcore/autotest/output.csv
 python_extras/example/output.csv
+*.egg
+gridlabd.egg-info/
+gldcore/link/python/dist/
+gldcore/link/python/gridlabd.egg-info/
+gridlabd.egg-*

Makefile.am

@@ -10,7 +10,8 @@
 # Authors:
 #   DP Chassin (dchassin@slac.stanford.edu)
 
-AM_INIT_AUTOMAKE = subdir-objects
+AM_INIT_AUTOMAKE = subdir-objects 
+AUTOMAKE_OPTIONS = -Wno-portability
 ACLOCAL_AMFLAGS = -I m4
 
 CC = $(PTHREAD_CC)
@@ -48,7 +49,7 @@ bin_PROGRAMS =
 bin_SCRIPTS =
 dist_doc_DATA =
 
-python_requirements = requirements.txt
+python_requirements = $(shell find $(top_srcdir) -name requirements.txt -print)
 
 docs_targets =
 
@@ -57,6 +58,7 @@ include $(top_srcdir)/third_party/Makefile.mk
 include $(top_srcdir)/gldcore/Makefile.mk
 include $(top_srcdir)/python_extras/Makefile.mk
 include $(top_srcdir)/cloud/Makefile.mk
+include $(top_srcdir)/utilities/Makefile.mk
 
 # approved modules
 include $(top_srcdir)/modules.mk
@@ -295,7 +297,7 @@ install-exec-hook: index python-requirements docs
 
 python-requirements: $(python_requirements)
 	@echo "Updating python requirements..."
-	@cat $(addprefix $(top_srcdir)/,$(python_requirements)) | python3 -m pip install -q -r /dev/stdin
+	@python3 -m pip install -q -r $(subst $(subst ,, ), -r ,$(python_requirements))
 
 reconfigure: distclean
 	autoreconf -isf && $(PWD)/configure --quiet

README.md

@@ -42,7 +42,21 @@ Normally on Linux and Mac OS X developers should use the `install.sh` script to
 host% git clone https://github.com/slacgismo/gridlabd gridlabd
 host% gridlabd/install.sh
 ~~~
-
+### AWS EC2 Installation 
+1) Set the path variable
+~~~
+host% sudo su
+host% export PATH=/usr/local/bin:$PATH
+~~~
+2) Change work dictionary and clone GitHub repository
+~~~
+host% cd /usr/local/src
+host% git clone https://github.com/slacgismo/gridlabd gridlabd
+~~~
+3) Run installation 
+~~~ 
+host% gridlabd/install.sh
+~~~
 To rebuild the source code and install again, use the `make system` command.  You can use parallel builds using the `make -j<nproc> system` command.
 
 If you have modified the branch name or version information, you must reconfigure your build using the `make reconfigure` command before using `make system`.

build-aux/setup-Linux-amzn-2.sh

@@ -1,7 +1,15 @@
 #!/bin/bash
 #
 # Install script for Amazon EC2 instance 
-#
+
+# Set environment variable in EC2.
+# In order to set the environment variable permanently, the system needs to be rebooted after the first time run this code.
+# The other way around is to run `export PATH=/usr/local/bin:$PATH` before run `./install.sh` in the command line. 
+# The temporary `PATH` will be generated and used for installation. 
+# The `PATH` will be set permanently because of running `echo "export PATH=/usr/local/bin:$PATH" >> /etc/profile.d/setVars.sh`
+
+echo "export PATH=/usr/local/bin:$PATH" >> /etc/profile.d/setVars.sh && \
+source /etc/profile.d/setVars.sh
 
 chmod -R 775 /usr/local
 chown -R root:adm /usr/local
@@ -16,8 +24,8 @@ yum -q install ncurses-devel -y
 yum -q install libcurl-devel -y
 
 # python3.9.x support needed as of 4.2
-if [ ! -x /usr/local/bin/python3 -o $(/usr/local/bin/python3 --version | cut -f-2 -d.) != "Python 3.9" ]; then
-	yum install openssl-devel bzip2-devel libffi-devel zlib-devel -q -y
+if [ ! -x /usr/local/bin/python3 -o "$(/usr/local/bin/python3 --version | cut -f2 -d.)" != "Python 3.9" ]; then
+	yum install openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel -q -y
 	cd /usr/local/src
 	curl https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz | tar xz
 	cd Python-3.9.6
@@ -30,17 +38,29 @@ if [ ! -x /usr/local/bin/python3 -o $(/usr/local/bin/python3 --version | cut -f-
 	ln -sf /usr/local/bin/idle3.9 /usr/local/bin/idle
 	ln -sf /usr/local/bin/pip3.9 /usr/local/bin/pip3
 	curl -sSL https://bootstrap.pypa.io/get-pip.py | /usr/local/bin/python3
-	/usr/local/bin/python3 pip -m install mysql-connector mysql-client matplotlib numpy pandas Pillow networkx
+	#/usr/local/bin/python3 pip -m install mysql-connector mysql-client matplotlib numpy pandas Pillow networkx
+	/usr/local/bin/python3 -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
+	/usr/local/bin/python3 -m pip install IPython censusdata
 fi
 
+
 # mono
 if [ ! -f /usr/bin/mono ]; then
-	rpmkeys --import "http://pool.sks-keyservers.net/pks/lookup?op=get&search=0x3fa7e0328081bff6a14da29aa6a19b38d3d831ef"
-	curl https://download.mono-project.com/repo/centos7-stable.repo > /etc/yum.repos.d/mono-centos7-stable.repo
-	yum -q install mono-devel -y
+	echo "Install mono"
+	cd ~
+	wget https://dl.fedoraproject.org/pub/fedora/linux/development/rawhide/Everything/x86_64/os/Packages/l/libpng15-1.5.30-13.fc35.x86_64.rpm
+	yum install -y ~/downloads/mono_dependencies/libpng15-1.5.30-13.fc35.x86_64.rpm
+	yum install yum-utils
+	rpm --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF"
+	yum-config-manager --add-repo http://download.mono-project.com/repo/centos/
+	yum clean all
+	yum makecache
+	yum install mono-complete -y
+	cd ~
+	rm -rf /tmp/mono_deps
 fi
 
-# natural_docs
+# # natural_docs
 if [ ! -x /usr/local/bin/natural_docs ]; then
 	cd /usr/local
 	curl https://www.naturaldocs.org/download/natural_docs/2.0.2/Natural_Docs_2.0.2.zip > natural_docs.zip
@@ -52,8 +72,15 @@ mono /usr/local/natural_docs/NaturalDocs.exe \$*' > /usr/local/bin/natural_docs
 	chmod a+x /usr/local/bin/natural_docs
 fi
 
-# converter support
-# cd /tmp
-# curl http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/m/mdbtools-0.7.1-3.el7.x86_64.rpm > mdbtools-0.7.1-3.el7.x86_64.rpm
-# rpm -Uvh mdbtools-0.7.1-3.el7.x86_64.rpm
+# # converter support
+
+echo "Install support"
+cd ~
+amazon-linux-extras install epel -y
+yum-config-manager --enable epel
 yum -q install mdbtools -y
+
+# # #latex
+echo "Install latex"
+yum -q install texlive -y
+

build-aux/setup-Linux-centos-8.sh

@@ -12,20 +12,32 @@ yum -q install which -y
 yum -q install svn -y
 
 # python3 support needed as of 4.2
-if [ ! -x /usr/local/bin/python3 -o $(/usr/local/bin/python3 --version) != "Python 3.9.0" ]; then
-	yum -q install openssl-devel bzip2-devel libffi-devel zlib-devel -y
+if [ ! -x /usr/local/bin/python3 -o "$(/usr/local/bin/python3 --version)" != "Python 3.9.6" ]; then
+	echo "install python 3.9.6"	
+
 	cd /usr/local/src
-	curl https://www.python.org/ftp/python/3.9.0/Python-3.9.0.tgz | tar xz
-	cd Python-3.9.0
+	yum install gcc openssl-devel bzip2-devel libffi-devel zlib-devel  xz-devel  -y
+	curl https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz | tar xz
+	cd Python-3.9.6
 	./configure --prefix=/usr/local --enable-optimizations --with-system-ffi --with-computed-gotos --enable-loadable-sqlite-extensions CFLAGS="-fPIC"
 	make -j $(nproc)
 	make altinstall
+
 	ln -sf /usr/local/bin/python3.9 /usr/local/bin/python3
 	ln -sf /usr/local/bin/python3.9-config /usr/local/bin/python3-config
 	ln -sf /usr/local/bin/pydoc3.9 /usr/local/bin/pydoc
 	ln -sf /usr/local/bin/idle3.9 /usr/local/bin/idle
 	ln -sf /usr/local/bin/pip3.9 /usr/local/bin/pip3
-	/usr/local/bin/python3 pip -m install mysql-connector matplotlib numpy pandas Pillow
+	# install python packages
+	/usr/local/bin/python3 -m pip install --upgrade pip
+<<<<<<< HEAD
+	/usr/local/bin/python3 -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd 
+=======
+	/usr/local/bin/python3 -m pip install mysql-connector matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd 
+>>>>>>> 13db67f9f90774b9ca73620eb394a1d9c37e5c08
+	/usr/local/bin/python3 -m pip install IPython censusdata
+
+
 fi
 
 # latex
@@ -49,8 +61,8 @@ fi
 
 # mono
 if [ ! -f /usr/bin/mono ]; then
-	rpmkeys --import "http://pool.sks-keyservers.net/pks/lookup?op=get&search=0x3fa7e0328081bff6a14da29aa6a19b38d3d831ef"
-	curl -s https://download.mono-project.com/repo/centos8-stable.repo | tee /etc/yum.repos.d/mono-centos8-stable.repo
+	rpmkeys --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF"
+	su -c 'curl https://download.mono-project.com/repo/centos8-stable.repo | tee /etc/yum.repos.d/mono-centos8-stable.repo'
 	yum -q install mono-devel -y
 fi
 

build-aux/setup-Linux-debian-10.sh

@@ -3,27 +3,38 @@
 # Install needed system tools
 # update first
 apt-get -q update
-apt-get -q install tzdata -y
 
-# install python 3.7
+
+apt-get install tzdata -y
+apt-get install curl -y
+apt-get install apt-utils -y
+
+# "Etc" will cause installation error
+if [ ! -f /etc/timezone -o "$(cat /etc/timezone | cut -f1 -d'/')" == "Etc" ]; then 
+	# get time zone from URL 
+	URL="https://ipapi.co/timezone"
+	response=$(curl -s -w "%{http_code}" $URL)
+	http_code=$(tail -n1 <<< "${response: -3}")  # get the last 3 digits
+	if [ $http_code == "200" ]; then 
+		time_zone=$(sed 's/.\{3\}$//' <<< "${response}") # remove the last 3 digits
+		echo "successful get timezone from  $URL , Set time zone as $time_zone"
+		ln -fs /usr/share/zoneinfo/$time_zone /etc/localtime
+	else 
+		echo "Can not get timezone from $URL , http_code is $http_code "
+		echo "Set default time zone as UTC/GMT. "
+		ln -fs /usr/share/zoneinfo/UTC/GMT /etc/localtime
+	fi
+
+	export DEBIAN_FRONTEND=noninteractive
+	apt-get install -y tzdata
+	dpkg-reconfigure --frontend noninteractive tzdata
+fi
+
 apt-get -q install software-properties-common -y
-apt install build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev curl -y
-cd /tmp
-curl -O https://www.python.org/ftp/python/3.7.3/Python-3.7.3.tar.xz
-tar -xf Python-3.7.3.tar.xz
-cd Python-3.7.3
-./configure --enable-optimizations --enable-shared CXXFLAGS="-fPIC"
-make -j10 altinstall
-cd /usr/local/bin
-ln -s python3.7 python3
-ln -s python3.7m-config python3-config
-ln -s pip3.7 pip3
-
-# install python libraries by validation
-pip3 -q install --upgrade pip
-pip -q install pandas matplotlib mysql-client Pillow
+apt install build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev -y
 
 # install system build tools needed by gridlabd
+
 apt-get -q install git -y
 apt-get -q install unzip -y
 apt-get -q install autoconf -y
@@ -33,8 +44,40 @@ apt-get -q install cmake -y
 apt-get -q install flex -y
 apt-get -q install bison -y
 apt-get -q install libcurl4-gnutls-dev -y
-apt-get -q install libncurses5-dev -y
 apt-get -q install subversion -y
+apt-get -q install util-linux -y
+apt-get install liblzma-dev -y
+apt-get install libbz2-dev -y
+apt-get install libncursesw5-dev -y
+apt-get install xz-utils -y
+
+# Install python 3.9.6
+# python3 support needed as of 4.2
+if [ ! -x /usr/local/bin/python3 -o "$(/usr/local/bin/python3 --version | cut -f2 -d.)" != "Python 3.9" ]; then
+	echo "install python 3.9.6"
+	cd /usr/local/src
+
+	curl https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz | tar xz
+	# tar xzf Python-3.9.6.tgz 
+	cd Python-3.9.6
+
+	./configure --prefix=/usr/local --enable-optimizations --with-system-ffi --with-computed-gotos --enable-loadable-sqlite-extensions CFLAGS="-fPIC"
+
+	make -j $(nproc)
+	make altinstall
+	/sbin/ldconfig /usr/local/lib
+	ln -sf /usr/local/bin/python3.9 /usr/local/bin/python3
+	ln -sf /usr/local/bin/python3.9-config /usr/local/bin/python3-config
+	ln -sf /usr/local/bin/pydoc3.9 /usr/local/bin/pydoc
+	ln -sf /usr/local/bin/idle3.9 /usr/local/bin/idle
+	ln -sf /usr/local/bin/pip3.9 /usr/local/bin/pip3
+	/usr/local/bin/python3 -m pip install --upgrade pip
+	/usr/local/bin/python3 -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
+	/usr/local/bin/python3 -m pip install IPython censusdata
+fi
+
+# install latex
+apt-get install texlive -y
 
 # doxgygen
 apt-get -q install gawk -y
@@ -51,12 +94,13 @@ if [ ! -x /usr/bin/doxygen ]; then
 	make install
 fi
 
-# mono
-apt-get -q install curl -y
+# # mono
+
 if [ ! -f /usr/bin/mono ]; then
 	cd /tmp
+	apt install apt-transport-https dirmngr gnupg ca-certificates -y
 	apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-	echo "deb http://download.mono-project.com/repo/ubuntu wheezy/snapshots/4.8.0 main" | tee /etc/apt/sources.list.d/mono-official.list
+	echo "deb https://download.mono-project.com/repo/debian stable-bustergrid main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list
 	apt-get -q update -y
 	apt-get -q install mono-devel -y
 fi