Release beauharnois-19

.github/workflows/develop.yml

@@ -16,6 +16,8 @@ jobs:
     - uses: actions/checkout@v2
     - name: Update python3 modules
       run: pip3 install -r requirements.txt
+    - name: Install openfido
+      run: curl -sL https://raw.githubusercontent.com/openfido/cli/main/install.sh | bash
     - name: Run autoconf
       run: autoreconf -isf
     - name: Configure build

.github/workflows/master.yml

@@ -16,6 +16,8 @@ jobs:
     - uses: actions/checkout@v2
     - name: Update python3 modules
       run: pip3 install -r requirements.txt
+    - name: Install openfido
+      run: curl -sL https://raw.githubusercontent.com/openfido/cli/main/install.sh | bash
     - name: Run autoconf
       run: autoreconf -isf
     - name: Configure build

build-aux/version.sh

@@ -8,7 +8,7 @@ PAT=`sed -En 's/#define REV_PATCH ([0-9]+).*/\1/p' $FIL | tr -d '\n'`
 PKG=`sed -En 's/#define PACKAGE "([a-z]+)".*/\1/p' $FIL | tr -d '\n'`
 NAM=`sed -En 's/#define PACKAGE_NAME "([-A-Za-z ]+)".*/\1/p' $FIL | tr -d '\n'`
 NUM=`git log --max-count=1 --pretty=format:"%ai" | cut -c 3,4,6,7,9,10`
-BRA=`git rev-parse --abbrev-ref HEAD`
+BRA=`git rev-parse --abbrev-ref HEAD | git rev-parse --abbrev-ref HEAD | tr -c A-Za-z0-9 _ | sed 's/_+/_/g;s/_$//;s/^_//'`
 GIT=`git --version | cut -f3 -d' '`
 SYS=`uname`
 HDW=`uname -m`

docs/Subcommand/Check.md

@@ -1,9 +1,9 @@
-[[/Subcommand/Check]] -- Subcommand to convert files from one format to another
+[[/Subcommand/Check]] -- Subcommand to perform a detailed check of the validity of a GridLAB-D model
 
 # Synopsis
 
 ~~~
-bash$ gridlabd convert [OPTIONS] FILE …
+bash$ gridlabd check [OPTIONS] FILE …
 ~~~
 
 # Description
@@ -17,45 +17,23 @@ objects in the model.  See the module's check routine for details.
 # Options
 
 ## `debug`
-
 ~~~
 	-d|--debug
 ~~~
-
 Enable python traceback output when problems are encountered
-
 ## `fix`
-
 ~~~
 	-f|--fix=FILE
 ~~~
-
 Fixes problems (if possible) are writes to the specified file
-
 ## `quiet`
-
 ~~~
 	-q|--quiet
 ~~~ 
-
 Suppress all non-error output
-
 ## `verbose`
-
 ~~~
 	-v|--verbose
 ~~~
-
 Enable additional output
 
-## `strict`
-
-~~~
-	-s|--strict
-~~~
-
-Causes warnings to be treated as errors with exit code 1.
-
-# See also
-
-* [[/Converters/README]]

docs/Subcommand/Job.md

@@ -5,13 +5,13 @@
 Shell:
 
 ~~~
-host% gridlabd job [-v|--verbose] [-q|--quiet] [-d|--debug] [-j|--jobfile JOBFILE] [-w|--workdir FOLDER] [-c|--configfile CONFIG] [-i|--interactive] FILE1 ...
+host% gridlabd job [-v|--verbose] [-q|--quiet] [-d|--debug] [-j|--jobfile JOBFILE] [-w|--workdir FOLDER] [-c|--configfile CONFIG] [-T|--threadcount NTHREADS] FILE1 ...
 ~~~
 
 GLM:
 
 ~~~
-#job [-v|--verbose] [-q|--quiet] [-d|--debug] [-j|--jobfile JOBFILE] [-w|--workdir FOLDER] [-c|--configfile CONFIG] [-i|--interactive] FILE1 ...
+#job [-v|--verbose] [-q|--quiet] [-d|--debug] [-j|--jobfile JOBFILE] [-w|--workdir FOLDER] [-c|--configfile CONFIG] [-T|--threadcount NTHREADS] FILE1 ...
 ~~~
 
 # Description
@@ -32,9 +32,9 @@ Specifies the name of the configuration file to use instead of `gridlabd-config.
 
 Enables debugging output.
 
-### `-i|--interactive`
+### `-T|--threadpool NTHREADS`
 
-Enables interactive display (requires curses library).
+Enables parallel processing of jobs using a threadpool using the specified number of threads.
 
 ### `-j|--jobfile JOBFILE`
 

docs/Subcommand/Json-get.md

@@ -0,0 +1,51 @@
+[[/Subcommand/Json-get]] -- Extract values from JSON data
+
+# Synopsis
+
+GLM:
+
+~~~
+#json-get [keys ...] [-k|--keys] [-j|--json|-r|--raw|-r|--csv] <INPUT >OUTPUT 
+~~~
+
+Shell:
+
+~~~
+$ gridlabd json-get [keys ...] [-k|--keys] [-j|--json|-r|--raw|-r|--csv] [<INPUT] [>OUTPUT] 
+~~~
+
+# Description
+
+The `json-get` subcommand extract elements from JSON data.  The `keys` are used to specify which elements are extracted.  Multiple keys will descend the JSON hierarchy recursively extracting data elements.
+
+By default the data is extracted in JSON format.  If `-r` or `--raw` is specified, the data will be extract in raw Python `str` format.  If `-c` or `--csv` is used, the data will be extract as comma-separated values. 
+
+# Caveat
+
+Not all data structures will format well as CSV.  In particular, nested dictionaries and lists may not yield valid or useful CSV data.  In general CSV formats are best suited for unnested dictionaries, e.g., `{"1":"a","2":"b","3":"c"}` yields
+
+~~~
+1,a
+2,b
+3,c
+~~~
+
+# Example
+
+~~~
+$ echo '{"id":"1","data":{"name":"test","value":"example"}}' | gridlabd json-get id
+1
+PC98798:gridlabd dchassin$ echo '{"id":"1","data":{"name":"test","value":"example"}}' | gridlabd json-get data
+{
+    "name": "test",
+    "value": "example"
+}
+$ echo '{"id":"1","data":{"name":"test","value":"example"}}' | gridlabd json-get data name
+test
+$ echo '{"id":"1","data":{"name":"test","value":"example"}}' | gridlabd json-get data value
+example
+ echo '{"id":"1","data":{"name":"test","value":"example"}}' | gridlabd json-get data --csv
+name,test
+value,example
+$ 
+~~~

gldcore/class.cpp

@@ -946,7 +946,7 @@ int class_define_map(CLASS *oclass, /**< the object class */
 			else if(proptype == PT_HAS_NOTIFY || proptype == PT_HAS_NOTIFY_OVERRIDE)
 			{
 				char notify_fname[128];
-				sprintf(notify_fname, "notify_%s_%s", prop->oclass->name, prop->name);
+				snprintf(notify_fname,sizeof(notify_fname)-1,"notify_%s_%s", prop->oclass->name, prop->name);
 				prop->notify = (FUNCTIONADDR)DLSYM(prop->oclass->module->hLib, notify_fname);
 				if(prop->notify == 0){
 					errno = EINVAL;
@@ -963,7 +963,7 @@ int class_define_map(CLASS *oclass, /**< the object class */
 			{
 				char tcode[32];
 				const char *ptypestr=class_get_property_typename(proptype);
-				sprintf(tcode,"%d",proptype);
+				snprintf(tcode,sizeof(tcode)-1,"%d",proptype);
 				if (strcmp(ptypestr,"//UNDEF//")==0)
 					ptypestr = tcode;
 				errno = EINVAL;
@@ -1091,7 +1091,7 @@ int class_define_enumeration_member(CLASS *oclass, /**< pointer to the class whi
 	KEYWORD *key = (KEYWORD*)malloc(sizeof(KEYWORD));
 	if (prop==NULL || key==NULL) return 0;
 	key->next = prop->keywords;
-	strncpy(key->name,member,sizeof(key->name));
+	strncpy(key->name,member,sizeof(key->name)-1);
 	key->value = value;
 	prop->keywords = key;
 	return 1;
@@ -1110,7 +1110,7 @@ int class_define_set_member(CLASS *oclass, /**< pointer to the class which imple
 	if (prop->keywords==NULL)
 		prop->flags |= PF_CHARSET; /* enable single character keywords until a long keyword is defined */
 	key->next = prop->keywords;
-	strncpy(key->name,member,sizeof(key->name));
+	strncpy(key->name,member,sizeof(key->name)-1);
 	key->name[sizeof(key->name)-1]='\0'; /* null terminate name in case is was too long for strncpy */
 	if (strlen(key->name)>1 && (prop->flags&PF_CHARSET)) /* long keyword detected */
 		prop->flags ^= PF_CHARSET; /* disable single character keywords */
@@ -1423,7 +1423,7 @@ static int buffer_write(char *buffer, /**< buffer into which string is written *
 		return 0;
 
 	va_start(ptr,format);
-	count = vsprintf(temp, format, ptr);
+	count = vsnprintf(temp, sizeof(temp)-1, format, ptr);
 	va_end(ptr);
 
 	if(count < len){

gldcore/cmdarg.cpp

@@ -62,12 +62,12 @@ DEPRECATED STATUS load_module_list(FILE *fd,int* test_mod_num)
 }
 DEPRECATED STATUS GldCmdarg::load_module_list(FILE *fd,int* test_mod_num)
 {
-	char mod_test[100];
+	char mod_test[1024];
 	char line[100];
 	while(fscanf(fd,"%s",line) != EOF)
 	{
 		printf("Line: %s",line);
-		sprintf(mod_test,"mod_test%d=%s",(*test_mod_num)++,line);
+		snprintf(mod_test,sizeof(mod_test)-1,"mod_test%d=%s",(*test_mod_num)++,line);
 		if (global_setvar(mod_test)!=SUCCESS)
 		{
 			output_fatal("Unable to store module name");
@@ -250,18 +250,20 @@ STATUS GldCmdarg::no_cmdargs(void)
 	char htmlfile[1024];
 	if ( global_autostartgui && find_file("gridlabd.htm",NULL,R_OK,htmlfile,sizeof(htmlfile)-1)!=NULL )
 	{
-		char cmd[1024];
+		char cmd[4096];
 
 		/* enter server mode and wait */
 #ifdef WIN32
 		if ( htmlfile[1]!=':' )
-			sprintf(htmlfile,"%s\\gridlabd.htm", global_workdir);
+		{
+			snprintf(htmlfile,sizeof(htmlfile)-1,"%s\\gridlabd.htm", global_workdir);
+		}
 		output_message("opening html page '%s'", htmlfile);
-		sprintf(cmd,"start %s file:///%s", global_browser, htmlfile);
+		snprintf(cmd,sizeof(cmd)-1,"start %s file:///%s", global_browser, htmlfile);
 #elif defined(MACOSX)
-		sprintf(cmd,"open -a %s %s", global_browser, htmlfile);
+		snprintf(cmd,sizeof(cmd)-1,"open -a %s %s", global_browser, htmlfile);
 #else
-		sprintf(cmd,"%s '%s' & ps -p $! >/dev/null", global_browser, htmlfile);
+		snprintf(cmd,sizeof(cmd)-1,"%s '%s' & ps -p $! >/dev/null", global_browser, htmlfile);
 #endif
 		IN_MYCONTEXT output_verbose("Starting browser using command [%s]", cmd);
 		if (my_instance->subcommand("%s",cmd)!=0)
@@ -1462,7 +1464,7 @@ int GldCmdarg::xsl(int argc, const char *argv[])
 				p_args[n_args] = p;
 			}
 		}
-		sprintf(fname,"gridlabd-%d_%d.xsl",global_version_major,global_version_minor);
+		snprintf(fname,sizeof(fname)-1,"gridlabd-%d_%d.xsl",global_version_major,global_version_minor);
 		output_xsl(fname,n_args,(const char**)p_args);
 		free(buffer);
 		return CMDOK;
@@ -1620,13 +1622,13 @@ int GldCmdarg::info(int argc, const char *argv[])
 {
 	if ( argc>1 )
 	{
-		char cmd[1024];
+		char cmd[4096];
 #ifdef WIN32
-		sprintf(cmd,"start %s \"%s%s\"", global_browser, global_infourl, argv[1]);
+		snprintf(cmd,sizeof(cmd)-1,"start %s \"%s%s\"", global_browser, global_infourl, argv[1]);
 #elif defined(MACOSX)
-		sprintf(cmd,"open -a %s \"%s%s\"", global_browser, global_infourl, argv[1]);
+		snprintf(cmd,sizeof(cmd)-1,"open -a %s \"%s%s\"", global_browser, global_infourl, argv[1]);
 #else
-		sprintf(cmd,"%s \"%s%s\" & ps -p $! >/dev/null", global_browser, global_infourl, argv[1]);
+		snprintf(cmd,sizeof(cmd)-1,"%s \"%s%s\" & ps -p $! >/dev/null", global_browser, global_infourl, argv[1]);
 #endif
 		IN_MYCONTEXT output_verbose("Starting browser using command [%s]", cmd);
 		if (my_instance->subcommand(cmd)!=0)
@@ -1864,7 +1866,7 @@ int GldCmdarg::mclassdef(int argc, const char *argv[])
         }
 
 	/* output the classdef */
-	count = sprintf(buffer,"struct('module','%s','class','%s'", modname, classname);
+	count = snprintf(buffer,sizeof(buffer)-1,"struct('module','%s','class','%s'", modname, classname);
 	for ( prop = oclass->pmap ; prop!=NULL && prop->oclass==oclass ; prop=prop->next )
 	{
 		char temp[1024];
@@ -1875,10 +1877,10 @@ int GldCmdarg::mclassdef(int argc, const char *argv[])
 		}
 		if ( value!=NULL )
 		{
-			count += sprintf(buffer+count, ",...\n\t'%s','%s'", prop->name, value);
+			count += snprintf(buffer+count,sizeof(buffer)-1-count,",...\n\t'%s','%s'", prop->name, value);
 		}
 	}
-	count += sprintf(buffer+count,");\n");
+	count += snprintf(buffer+count,sizeof(buffer)-1-count,");\n");
 	output_raw("%s",buffer);
         return CMDOK;
 }
@@ -2374,7 +2376,7 @@ STATUS GldCmdarg::load(int argc,const char *argv[])
 		{
 			CMDARG arg = main_commands[i];
 			char tmp[1024];
-			sprintf(tmp,"%s=",arg.lopt);
+			snprintf(tmp,sizeof(tmp)-1,"%s=",arg.lopt);
 			if ( ( arg.sopt && strncmp(*argv,"-",1)==0 && strcmp((*argv)+1,arg.sopt)==0 ) 
 			  || ( arg.lopt && strncmp(*argv,"--",2)==0 && strcmp((*argv)+2,arg.lopt)==0 ) 
 			  || ( arg.lopt && strncmp(*argv,"--",2)==0 && strncmp((*argv)+2,tmp,strlen(tmp))==0 ) )

gldcore/convert.cpp

@@ -986,7 +986,7 @@ int convert_to_object(const char *buffer, /**< a pointer to the string buffer */
 		*target = NULL;
 		return 1;
 	}
-	else if ( sscanf(buffer,"\"%[^\"]\"",oname) == 1 || (strchr(buffer,':') == NULL && strncpy(oname,buffer,sizeof(oname))) )
+	else if ( sscanf(buffer,"\"%[^\"]\"",oname) == 1 || (strchr(buffer,':') == NULL && strncpy(oname,buffer,sizeof(oname)-1)) )
 	{
 		oname[sizeof(oname)-1]='\0'; /* terminate unterminated string */
 		*target = object_find_name(oname);
@@ -1501,7 +1501,7 @@ int convert_to_struct(const char *buffer, void *data, PROPERTY *structure)
 	{
 		return -1;
 	}
-	strncpy(temp,buffer+1,sizeof(temp));
+	strncpy(temp,buffer+1,sizeof(temp)-1);
 	char *item = NULL;
 	char *last = NULL;
 	while ( (item=strtok_s(item?NULL:temp,";",&last)) != NULL )

gldcore/debug.cpp

@@ -222,7 +222,7 @@ static const char *get_objname(OBJECT *obj)
 {
 	static char buf[1024];
 	if (obj->name) return obj->name;
-	sprintf(buf,"%s:%d", obj->oclass->name, obj->id);
+	snprintf(buf,sizeof(buf)-1,"%s:%d", obj->oclass->name, obj->id);
 	return buf;
 }
 
@@ -241,7 +241,7 @@ static STATUS exec(const char *format,...)
 	char cmd[1024];
 	va_list ptr;
 	va_start(ptr,format);
-	vsprintf(cmd,format,ptr);
+	vsnprintf(cmd,sizeof(cmd)-1,format,ptr);
 	va_end(ptr);
 	output_debug("Running '%s'", cmd);
 	return system(cmd)==0?SUCCESS:FAILED;
@@ -426,7 +426,7 @@ static int exec_add_watchpoint(OBJECT *obj, /**< the object being watched */
 
 static void list_object(OBJECT *obj, PASSCONFIG pass)
 {
-	char details[132] = "";
+	char details[4096] = "";
 	char buf1[64],buf2[64],buf3[64];
 	if (list_unnamed==0 && obj->name==NULL)
 		return;
@@ -438,7 +438,7 @@ static void list_object(OBJECT *obj, PASSCONFIG pass)
 	{
 		char valid_to[64] = "";
 		convert_from_timestamp(obj->valid_to,valid_to,sizeof(valid_to));
-		sprintf(details,"%s %c%c%c %s/%s/%d ", valid_to,
+		snprintf(details,sizeof(details)-1,"%s %c%c%c %s/%s/%d ", valid_to,
 			obj->flags&OF_RECALC?'c':'-', obj->flags&OF_RERANK?'r':'-', obj->flags&OF_FOREIGN?'f':'-',
 			obj->oclass->module->name, obj->oclass->name, obj->id);
 	}