Fix build warnings

.github/workflows/develop.yml

@@ -15,24 +15,18 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    # this is to fix GIT not liking owner of the checkout dir
-    - name: Set ownership
-      run: |
-        chown -R $(id -u):$(id -g) $PWD
-
-    - name: Update pip
-      run: python3 -m pip install --upgrade pip
-
-    - name: install gridlabd
+    - name: Build gridlabd
       run: |
+        set +x
+        chown -R $(id -u):$(id -g) $PWD /github/home
         autoreconf -isf
         ./configure
-        make -j$(($(nproc)*3)) system
+        make -j$(($(nproc)*3)) python-install system
 
     - name: Install openfido
       run: curl -sL https://raw.githubusercontent.com/openfido/cli/main/install.sh | bash
 
-    - name: Validate build
+    - name: Validate system
       run: |
         gridlabd -D keep_progress=TRUE -T 0 --validate -D github_actions=yes || ( utilities/save_validation_errors ; false )
 

.gitignore

@@ -140,6 +140,10 @@ fault_check.txt
 # files that should be ignored by git
 *_ignore.*
 
+# requirements files
+requirements.txt
+python/venv
+
 # specific outputs that should be ignored
 python/example.png
 models/ieee123/config/local.glm

Makefile.am

@@ -10,6 +10,14 @@
 # Authors:
 #   DP Chassin (dchassin@slac.stanford.edu)
 
+# !!! must match version specified in configure.ac
+PYVER = 3.10
+PYCFLAGS = $(shell python3.10-config --cflags)
+PYLDFLAGS = $(shell python3.10-config --ldflags)
+
+SYSPYTHON = python$(PYVER)
+PYTHONCONFIG = $(SYSPYTHON)-config
+
 AM_INIT_AUTOMAKE = subdir-objects 
 AUTOMAKE_OPTIONS = -Wno-portability
 ACLOCAL_AMFLAGS = -I m4
@@ -18,27 +26,30 @@ CC = $(PTHREAD_CC)
 CXX = $(PTHREAD_CXX)
 
 AM_CPPFLAGS = -Wall -fPIC
-AM_CPPFLAGS += -I$(top_srcdir)/source -I$(top_srcdir)/build-dir/source
+AM_CPPFLAGS += -I$(top_srcdir)/source #-I$(top_srcdir)/build-dir/source
 AM_CPPFLAGS += $(GLD_CPPFLAGS)
+AM_CPPFLAGS += $(PYCFLAGS)
 
 AM_CFLAGS = -Wall
-AM_CFLAGS +=  -I$(top_srcdir)/source -I$(top_srcdir)/build-dir/source
+AM_CFLAGS +=  -I$(top_srcdir)/source #-I$(top_srcdir)/build-dir/source
 AM_CFLAGS += $(PTHREAD_CFLAGS)
-AM_CFLAGS += $(GLD_CFLAGS)  -I$(top_srcdir)/build-dir/source
+AM_CFLAGS += $(GLD_CFLAGS)  #-I$(top_srcdir)/build-dir/source
+AM_CFLAGS += $(PYCFLAGS)
 
 AM_CXXFLAGS = -Wall -fPIC
 AM_CXXFLAGS += $(PTHREAD_CFLAGS)
-AM_CXXFLAGS += $(GLD_CXXFLAGS) -I$(top_srcdir)/build-dir/source
+AM_CXXFLAGS += $(GLD_CXXFLAGS) #-I$(top_srcdir)/build-dir/source
+AM_CXXFLAGS += $(PYCFLAGS)
 
-AM_LDFLAGS =
+AM_LDFLAGS = $(PYLDFLAGS)
 AM_LDFLAGS += -no-undefined
 AM_LDFLAGS += -module
 AM_LDFLAGS += -avoid-version
 AM_LDFLAGS += -export-dynamic
-AM_LDFLAGS += $(GLD_LDFLAGS)
+AM_LDFLAGS += $(GLD_LDFLAGS) -lpython$(PYVER)
 
 # makefile fragments populate these variables
-BUILT_SOURCES =
+BUILT_SOURCES = 
 CLEANFILES =
 EXTRA_DIST =
 dist_pkgdata_DATA =
@@ -49,8 +60,6 @@ bin_PROGRAMS =
 bin_SCRIPTS =
 dist_doc_DATA =
 
-python_requirements = $(shell find $(top_srcdir) -name requirements.txt -print)
-
 docs_targets =
 
 # required modules
@@ -71,6 +80,55 @@ include $(top_srcdir)/modules.mk
 # custom modules (see ./customize)
 include $(top_srcdir)/custom.mk
 
+BUILT_SOURCES += python-requirements
+
+#
+# Post install report
+#
+
+install-exec-hook: | index docs
+	@cp ${top_srcdir}/{COPYRIGHT,LICENSE} $(prefix)
+	@echo ""
+	@echo "Install complete. Here are some useful commands now:"
+	@echo ""
+	@echo "To include this version in your command path:"
+	@echo "  export PATH='$(DESTDIR)$(bindir):$$PATH'"
+	@echo ""
+	@echo "To run this version directly without using the command path:"
+	@echo "  $(DESTDIR)$(bindir)/gridlabd"
+	@echo ""
+	@echo "To make this version the default for all users on this system:"
+	@echo "  $(DESTDIR)$(bindir)/gridlabd version set $(shell basename $(exec_prefix))"
+	@echo ""
+
+#
+# Python setup
+#
+
+PYENV=$(bindir)/pkgenv
+PYBIN=$(PYENV)/bin
+PYLIB=$(PYENV)/lib
+PYINC=$(PYENV)/include
+ENVPYTHON=$(PYBIN)/python$(PYVER)
+PYACTIVATE=source $(PYBIN)/activate
+
+python-requirements: | $(PYENV) 
+
+# NB: it's ok to use the runner's python to generate the requirements.txt file
+requirements.txt: python/requirements.csv
+	@python3 -m pip -q install pandas==2.0.0
+	@python3 python/requirements.py $< >$@
+
+$(PYENV): requirements.txt
+	@echo "Processing python requirements..."
+	@mkdir -p $(PYBIN) $(PYLIB) $(PYINC)
+	@$(SYSPYTHON) -m venv --symlinks $(PYENV)
+	@$(ENVPYTHON) -m pip install --upgrade pip
+	@$(ENVPYTHON) -m pip install pandas==2.0.0
+	@$(ENVPYTHON) python/requirements.py --buildenv | bash
+	@$(ENVPYTHON) -m pip install -r requirements.txt 
+	@touch $(PYENV)
+
 # for doxygen commands
 include $(top_srcdir)/aminclude.mk
 
@@ -285,24 +343,9 @@ distclean-local: python-clean
 libtool: $(LIBTOOL_DEPS)
 	$(SHELL) $(PWD)/config.status libtool > /dev/null
 
-install-exec-hook: index python-requirements docs
-	@cp ${top_srcdir}/{COPYRIGHT,LICENSE} $(prefix)
-	@echo ""
-	@echo "Install complete. Here are some useful commands now:"
-	@echo ""
-	@echo "To include this version in your command path:"
-	@echo "  export PATH='$(DESTDIR)$(bindir):$$PATH'"
-	@echo ""
-	@echo "To run this version directly without using the command path:"
-	@echo "  $(DESTDIR)$(bindir)/gridlabd"
-	@echo ""
-	@echo "To make this version the default for all users on this system:"
-	@echo "  $(DESTDIR)$(bindir)/gridlabd version set $(shell basename $(exec_prefix))"
-	@echo ""
-
-python-requirements: $(python_requirements)
-	@echo "Updating python requirements..."
-	@$(bindir)/pkgenv/bin/python3 -m pip install -q -r $(subst $(subst ,, ), -r ,$(python_requirements))
+#
+# Run this after changes that affect build-aux/version.sh output
+#
 
 reconfigure: distclean
 	autoreconf -isf && $(PWD)/configure --quiet

build-aux/version.sh

@@ -36,6 +36,15 @@ case $1 in
         echo "$PKG" ;;
     --package-name | -P)
         echo "$NAM" ;;
+    --sysspec )
+        if [ "$(uname -s)" == "Darwin" ]; then
+            echo darwin$(uname -r | cut -f1 -d.)-$(uname -m)
+        elif [ -f "/etc/os-release" ]; then
+            echo $(grep ^ID= /etc/os-release | cut -f2 -d= | tr -d '"')-$(grep ^VERSION_ID= /etc/os-release | cut -f2 -d= | tr -d '"')-$(uname -m)
+        else
+            echo $(uname -s)$(uname -r)-$(uname -m)
+        fi
+        ;;
     --help | -h)
         echo "Syntax: build-aux/version.sh <option>
 Options:
@@ -47,5 +56,6 @@ Options:
   --package|-p      output package PACKAGE
   --package-name|-P output package name PACKAGE_NAME
   --name|-m         output install name REV_MAJOR.REV_MINOR.REV_PATCH-NUMBER-BRANCH
+  --sysspec         output system specification OSVER-ARCH
   --help|-h         output this help" ;;
 esac

configure.ac

@@ -52,6 +52,7 @@ AC_CONFIG_AUX_DIR([build-aux])
 ###############################################################################
 # Init automake
 ###############################################################################
+
 AM_INIT_AUTOMAKE([foreign subdir-objects -Wall])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES])
 
@@ -578,103 +579,16 @@ BINDIR=$DSTDIR/bin
 LIBDIR=$DSTDIR/lib/gridlabd
 ETCDIR=$DSTDIR/share/gridlabd
 DOCDIR=$DSTDIR/share/doc/gridlabd
-ENVDIR=$BINDIR/pkgenv
-PYDIR=$ENVDIR/bin
-PYEXE=3.10
-
-echo "os: $host_os"
-echo "cpu: $host_cpu"
 
-# Verify python version
-if test ! -d "$BINDIR/pkgenv"; then
-    mkdir -p "$BINDIR"
-fi
+PYTHON_REQ=3.10
 
-if echo "$host_os" | grep '^linux' >/dev/null; then
-    # Check for Debian or Ubuntu
-    if grep -E '^ID=(debian|ubuntu)$' /etc/*-release; then
-        # Check for Debian 10
-        if grep -E '^VERSION_ID="10"' /etc/*-release; then
-            echo "Detected Debian 10 operating system"
-            # Add Debian 10-specific configuration here
-            /usr/local/bin/python$PYEXE -m venv "$BINDIR/pkgenv"
-            "$PYDIR/python3" -m pip install --upgrade pip
-            "$PYDIR/python3" -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
-            "$PYDIR/python3" -m pip install IPython censusdata
-            "$PYDIR/python3" -m pip install GDAL==2.4.0
-            "$PYDIR/python3" -m pip install rasterio==1.2.10
-        elif grep -E '^VERSION_ID="9"' /etc/*-release; then
-            echo "Detected Debian 9 operating system"
-            # Add Debian 9-specific configuration here
-            /usr/local/bin/python$PYEXE  -m venv "$BINDIR/pkgenv"
-            "$PYDIR/python3" -m pip install --upgrade pip
-            "$PYDIR/python3" -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
-            "$PYDIR/python3" -m pip install IPython censusdata
-        else
-            echo "Detected Debian or Ubuntu operating system"
-            # Add Debian/Ubuntu-specific configuration here
-            /usr/local/bin/python$PYEXE  -m venv "$BINDIR/pkgenv"
-            "$PYDIR/python3" -m pip install --upgrade pip
-            "$PYDIR/python3" -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
-            "$PYDIR/python3" -m pip install IPython censusdata
-            "$PYDIR/python3" -m pip install GDAL==3.0.4
-            "$PYDIR/python3" -m pip install rasterio==1.2.10
-        fi
-
-        if test ! -d "$BINDIR/pkgenv/include/python*"; then
-            ln -sF /usr/local/bin/python${PYEXE}-config $BINDIR/pkgenv/bin/python3-config
-            ln -sF /usr/local/lib/*${PYEXE}* $BINDIR/pkgenv/lib
-            ln -sF /usr/local/include/python${PYEXE} $BINDIR/pkgenv/include
-        fi
-
-    else
-        echo "Unsupported Linux distribution"
-        exit 1
-    fi
-elif echo "$host_os" | grep '^darwin' >/dev/null; then
-    if echo "$host_cpu" | grep -E 'i386|x86_64'; then
-        echo "Detected Intel Mac operating system"
-        # Add Intel Mac-specific configuration here
-        /usr/local/bin/python$PYEXE -m venv "$BINDIR/pkgenv"
-
-        ln -sF /usr/local/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/bin/python$PYEXE-config $BINDIR/pkgenv/bin/python3-config
-        ln -sF /usr/local/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/include/python$PYEXE $BINDIR/pkgenv/include
-        ln -sF /usr/local/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/lib/*${PYEXE}* $BINDIR/pkgenv/lib
-
-        $PYDIR/python3 -m ensurepip --upgrade
-        $PYDIR/python3 -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
-        $PYDIR/python3 -m pip install build
-        $PYDIR/python3 -m pip install pyproj wheel
-
-    elif echo "$host_cpu" | grep 'aarch64'; then
-        echo "Detected arm64 Mac operating system"
-        # Add arm64 Mac-specific configuration here
-        /opt/homebrew/bin/python$PYEXE -m venv "$BINDIR/pkgenv"
-
-        ln -sF /opt/homebrew/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/bin/python$PYEXE-config $BINDIR/pkgenv/bin/python3-config
-        ln -sF /opt/homebrew/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/include/python$PYEXE $BINDIR/pkgenv/include
-        ln -sF /opt/homebrew/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/lib/*${PYEXE}* $BINDIR/pkgenv/lib
-
-        $PYDIR/python3 -m ensurepip --upgrade
-        $PYDIR/python3 -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
-        $PYDIR/python3 -m pip install build
-        $PYDIR/python3 -m pip install pyproj wheel
-    else
-        echo "Unknown CPU architecture on Mac operating system"
-        exit 1
-    fi
-else
-    echo "Unsupported operating system: $host_os"
-    exit 1
-fi
-
-HAVE_PYTHON="$($BINDIR/pkgenv/bin/python3 --version | cut -f2 -d' ')"
-PYTHON_EXEC="$($BINDIR/pkgenv/bin/python3 -c "import os; print(os.path.realpath('$BINDIR/pkgenv/bin/python3'))")"
-PYTHON_VER=$(echo $HAVE_PYTHON | cut -f1-2 -d.)
+AM_PATH_PYTHON([$PYTHON_REQ])
+PYTHON=$(which python$PYTHON_REQ)
+HAVE_PYTHON="$($PYTHON --version | cut -f2 -d' ')"
 AM_CONDITIONAL([HAVE_PYTHON], [test ! "x$HAVE_PYTHON" = "xno"])
 AS_IF([test "x$HAVE_PYTHON" != "xno" -a "$(uname)" == "Darwin" ],
-  [PYTHON_LIB="-lpython${PYTHON_VER} -shared -Wl-undefined,dynamic_lookup"],
-  [PYTHON_LIB="-lpython${PYTHON_VER}"])
+  [PYTHON_LIB="-lpython${PYTHON_REQ} -shared -Wl-undefined,dynamic_lookup"],
+  [PYTHON_LIB="-lpython${PYTHON_REQ}"])
 
 ###############################################################################
 # The End
@@ -690,7 +604,7 @@ AC_MSG_RESULT([
     Version: .................... $VERSION
     Branch: ..................... $PACKAGE_ORIGIN
     Install prefix: ............. ${DSTDIR/^$PWD/.}
-    Python Location: ............ $PYTHON_EXEC
+    Python binary: .............. $PYTHON
 
   Dependencies:
 
@@ -708,6 +622,7 @@ AC_MSG_RESULT([
 
   Build flags:
 
+    CFLAGS: ..................... ${CFLAGS:-(none)}
     CXXFLAGS: ................... ${CXXFLAGS:-(none)}
     LDFLAGS: .................... ${LDFLAGS:-(none)}
 
@@ -744,18 +659,9 @@ AC_MSG_RESULT([
 
         $DSTDIR/bin/$PACKAGE version set
 
-    Developer flags:
-    CFLAGS ........ "$CFLAGS -D PYTHON_EXEC='\"$PYTHON_EXEC\"'"
-    CXXFLAGS ...... "$($BINDIR/pkgenv/bin/python3-config --cflags) $CXXFLAGS -D PYTHON_EXEC='\"$PYTHON_EXEC\"'"
-    LDFLAGS ....... "$($BINDIR/pkgenv/bin/python3-config --ldflags) $PYTHON_LIB $LDFLAGS "
-
 -----------------------------------------------------------------------
 ])
 
-CFLAGS="$CFLAGS -D PYTHON_EXEC='\"$PYTHON_EXEC\"'"
-CXXFLAGS="$($BINDIR/pkgenv/bin/python3-config --cflags) $CXXFLAGS -D PYTHON_EXEC='\"$PYTHON_EXEC\"'"
-LDFLAGS="$($BINDIR/pkgenv/bin/python3-config --ldflags) $PYTHON_LIB $LDFLAGS "
-
 AC_CONFIG_FILES([Makefile])
 AC_CONFIG_FILES([gridlabd.spec])
 AC_CONFIG_FILES([source/gridlabd], [chmod +x source/gridlabd])

converters/Makefile.mk

@@ -2,8 +2,6 @@
 # INPUT CONVERTERS
 #
 
-python_requirements += $(top_srcdir)/converters/requirements.txt
-
 # csv -> glm
 dist_pkgdata_DATA += converters/csv-ami2glm-ceus.py
 dist_pkgdata_DATA += converters/csv-ami2glm-rbsa.py

converters/mdb-cyme2glm.py

@@ -80,7 +80,7 @@
 	"DEBUG" : ["False"],
 	"QUIET" : ["False"],
 	"VERBOSE" : ["False"],
-	}).transpose().set_axis(["value"],axis=1,inplace=0)
+	}).transpose().set_axis(["value"],axis=1)
 config.index.name = "name" 
 
 cyme_phase_name = {0:"ABCN", 1:"A", 2:"B", 3:"C", 4:"AB", 5:"AC", 6:"BC", 7:"ABC"} # CYME phase number -> phase names

geodata/autotest/unittest.sh

@@ -4,7 +4,7 @@ DIR=${0/unittest.sh/..}
 N=0
 for F in $DIR/*.py; do
 	echo '*** '"$(basename $F) unittest"' ***' > /tmp/$$_$N.err
-	( $GLD_BIN/pkgenv/bin/python3 $F -q 2>>/tmp/$$_$N.err && rm /tmp/$$_$N.err ) &
+	( gridlabd python $F -q 2>>/tmp/$$_$N.err && rm /tmp/$$_$N.err ) &
 	N=$(($N+1))
 done
 wait