Fix geodata table input (#970)

* Fix problems with table2glm converter when using geodata

* Fix json.date_format type

* Add --json.indent option for geodata output JSON format

* Update tutorial files

* Fix folium so it generates output even when no location data is present

* Fix output of object name

* Update Geodata.ipynb

* Update Geodata.md

* Update example_1.glm

* Update example_2.glm

* Update example_3.glm

* Update example_4.glm

* Create poles.csv

* Create .gitignore

* Removing ignored files

* Delete example.html

* Update .gitignore

* Fixed geodata tutorial example 6

* Create example_5.glm

* Update tutorial

* Update .gitignore

* Tutorial cleanup

* Update csv-table2glm-object.py

* Update example_2.glm

* Update json2html.py

* Update .gitignore

* Update example_3.glm

* Fix autotest errors

* Update json2html.py

* Fix import converter (#976)

* Update version.h
* Fix import converter
* Delete table2glm_input.glm
Co-authored-by: Alyona Teyber <Ivanova.alyona5@gmail.com>

* Fixing the table2glm converter and adding a more robust testing input

* Update test_table2glm.glm

* Update Geodata.ipynb

* Update test_geodata.glm

* Update Geodata.md

* Update test_geodata.txt

Co-authored-by: Alyona Teyber <Ivanova.alyona5@gmail.com>

.gitignore

@@ -175,3 +175,4 @@ gldcore/converters/autotest/table2glm_input_noname.glm
 gldcore/converters/autotest/table2glm_input.glm
 gldcore/geodata/docs/geodata.conf
 gldcore/gridlabd.in
+geodata.conf

gldcore/converters/autotest/table2glm_input.csv

@@ -1,2 +1,3 @@
-class,name ,wind_speed,temperature
-climate,weather,4 m/s,95 degF
+name,wind_speed,temperature,phases,nominal_voltage,class
+weather,4 m/s,95 degF,,,climate
+node1,,,AB,120,

gldcore/converters/autotest/test_table2glm.glm

@@ -1,5 +1,9 @@
 module assert;
 
+#ifexist "../table2glm_input.csv"
+#define DIR=..
+#endif
+
 clock 
 {
 	starttime "2019-01-01 00:00:00";
@@ -9,7 +13,7 @@ clock
 module powerflow;
 module climate;
 
-#input "../table2glm_input.csv" -f "table" -t "object"
+#input "${DIR:-.}/table2glm_input.csv" -f "table" -t "object" -C "powerflow.node"
 
 object assert {
 	parent "weather";

gldcore/converters/autotest/test_table2glm_err.glm

@@ -1,5 +1,9 @@
 module assert;
 
+#ifexist "../table2glm_input_noclass.csv"
+#define DIR=..
+#endif
+
 clock 
 {
 	starttime "2019-01-01 00:00:00";
@@ -9,7 +13,7 @@ clock
 module powerflow;
 module climate;
 
-#input "../table2glm_input_noclass.csv" -f "table" -t "object"
+#input "${DIR:-.}/table2glm_input_noclass.csv" -f "table" -t "object"
 
 object assert {
 	parent "weather";

gldcore/converters/autotest/test_table2glm_noclass.glm

@@ -1,5 +1,9 @@
 module assert;
 
+#ifexist "../table2glm_input_noclass.csv"
+#define DIR=..
+#endif
+
 clock 
 {
 	starttime "2019-01-01 00:00:00";
@@ -9,7 +13,7 @@ clock
 module powerflow;
 module climate;
 
-#input "../table2glm_input_noclass.csv" -f "table" -t "object" -C "climate"
+#input "${DIR:-.}/table2glm_input_noclass.csv" -f "table" -t "object" -C "climate"
 
 object assert {
 	parent "weather";

gldcore/converters/autotest/test_table2glm_noname.glm

@@ -1,5 +1,9 @@
 module assert;
 
+#ifexist "../table2glm_input_noname.csv"
+#define DIR=..
+#endif
+
 clock 
 {
 	starttime "2019-01-01 00:00:00";
@@ -9,16 +13,21 @@ clock
 module powerflow;
 module climate;
 
-#input "../table2glm_input_noname.csv" -f "table" -t "object"
+global double wind_speed[m/s] 4;
+global double temperature[degF] 95;
+
+#input "${DIR:-.}/table2glm_input_noname.csv" -f "table" -t "object"
 
+#for OBJ in ${FIND class=climate} 
 object assert {
-	parent "climate_1";
+	parent ${OBJ};
 	target "wind_speed";
 	value 4 m/s;
 }
 
 object assert {
-	parent "climate_1";
+	parent ${OBJ};
 	target "temperature";
 	value 95 degF;
-}
+}
+#done

gldcore/converters/csv-table2glm-object.py

@@ -1,38 +1,44 @@
-"""Convert Anticipation data sheet in CSV to an pole vulnerability GLM template parameter set
+"""Convert generable CSV table into a list of GLM objects
+Valid options:
+  class:   specifies the default class to use when class is not specified in the table
 """
 import csv
 import re
 import sys, getopt
 
 def error(msg):
-    print(f'ERROR   {msg}')
+    print(f'ERROR [converters/csv-table2glm-object]: {msg}',file=sys.stderr)
     sys.exit(1)
 
+def warning(msg):
+    print(f'WARNING [converters/csv-table2glm-object]: {msg}',file=sys.stderr)
+
 def convert (p_configuration_in, p_configuration_out, options={} ) : 
-	with open(p_configuration_in, newline='') as csvfile:
+	with open(p_configuration_in, "r", newline='') as csvfile:
 		configurations = csv.reader(csvfile, delimiter=',')
-		p_config_out = open(p_configuration_out, "a")
-		p_config_out.truncate(0)
-		p_config_out.write("// Objects \n")
-		module_list = []		
+		p_config_out = open(p_configuration_out, "w")
+		p_config_out.write(f"// {__file__} {p_configuration_in} {p_configuration_out} {options} \n")
 		for i, row in enumerate(configurations):
 			if i == 0 : 
 				headers = row
+				if "class" in headers:
+					class_index = headers.index("class")
+					if options:
+						warning(f"class data overrides option from file")
+
+				else:
+					class_index = None
 			else : 
 				if "class" not in headers and not options: 
 					error("No class name found, please edit your CSV to include class or add -C <class name> to your input command")
 				else : 
 					class_index=headers.index("class")
 				if not row[class_index] : 
+					p_config_out.write(f"object {options['class']} ")
 					class_name = options['class']
 				if row[class_index] : 
+					p_config_out.write(f"object {row[class_index]} ")
 					class_name = row[class_index]
-				if "." in class_name :
-					class_spec = class_name.split(".")[0]
-					if class_spec not in module_list: 
-						p_config_out.write(f"module {class_spec};\n")
-						module_list.append(class_spec)
-				p_config_out.write(f"object {class_name} ")
 				p_config_out.write("{ \n")
 				for j,value in enumerate (row) : 
 					if j!=class_index and headers[j] : 
@@ -45,4 +51,4 @@ def convert (p_configuration_in, p_configuration_out, options={} ) :
 								else : 
 									p_config_out.write("\t" + headers[j].strip() + " " + "\"" +value + "\"" + ";\n")
 				p_config_out.write("}\n")
-		p_config_out.close()
+		p_config_out.close()

gldcore/converters/csv2glm.py

@@ -21,7 +21,8 @@ def help():
     print(f'  -f|--from      : [REQUIRED] input {config["input"]} data type')
     print(f'  -t|--type      : [REQUIRED] output {config["output"]} data type')
     print(f'  -p|--property  : [OPTIONAL] property option')
-    print(f'  -C|--class     : [OPTIONAL] optional specification for class definition in the table converter')
+    print(f'  -C|--class     : [OPTIONAL] default class definition when generating GLM objects')
+    print(f'  -M|--module    : [OPTIONAL] default module definition when generating GLM objects')
 
 def error(msg):
     print(f'ERROR    [{config["input"]}2{config["output"]}]: {msg}')
@@ -33,7 +34,7 @@ def error(msg):
 output_type = None
 options = {}
 
-opts, args = getopt.getopt(sys.argv[1:],"hci:o:f:t:p:C:",["help","config","ifile=","ofile=","from=","type=","property=","class="])
+opts, args = getopt.getopt(sys.argv[1:],"hci:o:f:t:p:C:M:",["help","config","ifile=","ofile=","from=","type=","property=","class="])
 
 if not opts : 
     help()
@@ -59,6 +60,8 @@ def error(msg):
         options[prop[0]] = prop[1]
     elif opt in ("-C","--class"):
         options["class"] = arg.strip()
+    elif opt in ("-M","--module"):
+        options["module"] = arg.strip()
     else:
         error(f"{opt}={arg} is not a valid option");
 

gldcore/converters/json2html.py

@@ -1,4 +1,14 @@
-"""Convert GLM to HTML Folium interactive map"""
+"""Convert GLM to HTML Folium interactive map
+
+Object properties that affect map generation:
+
+  hidden
+  map_color
+  map_weight (link only)
+  map_opacity (link only)
+
+If defined in the object will be created (if hidden not TRUE) using these properties.
+"""
 import os, sys, getopt
 import json 
 import folium
@@ -97,11 +107,15 @@ def help():
     lats = numpy.array(lats)
     lons = numpy.array(lons)
 
-    if zoomlevel == 'auto':
-        map = folium.Map(location=[lats.mean(),lons.mean()],tiles=tiles)
-        map.fit_bounds([[lats.min(),lons.min()],[lats.max(),lons.max()]])
-    else:
-        map = folium.Map(location=[lats.mean(),lons.mean()],tiles=tiles,zoom_start=zoomlevel)
+    try:
+        if zoomlevel == 'auto':
+            map = folium.Map(location=[lats.mean(),lons.mean()],tiles=tiles)
+            map.fit_bounds([[lats.min(),lons.min()],[lats.max(),lons.max()]])
+        else:
+            map = folium.Map(location=[lats.mean(),lons.mean()],tiles=tiles,zoom_start=zoomlevel)
+    except:
+        map = folium.Map(tiles=tiles,zoom_start=zoomlevel)
+        pass
     try:
         if cluster_ok:
             cluster = MarkerCluster().add_to(map)
@@ -115,13 +129,16 @@ def help():
         popup = get_popup(name,tag)
         oclass = tag['class']
         color = get_color(tag)
+        if "hidden" in tag.keys() and tag["hidden"] == "TRUE":
+            continue
         if icon_prefix in icons.keys():
             if oclass in icons[icon_prefix].keys():
                 icon = folium.Icon(icon=icons[icon_prefix][oclass],color=color,prefix=icon_prefix)
             else:
                 icon = None
         else:
             icon = folium.Icon(color=color)
+        obj = None
         try: # attempt to handle as a link
             from_name = tag['from']
             from_obj = data['objects'][from_name]
@@ -131,24 +148,34 @@ def help():
             to_obj = data['objects'][to_name]
             lat1 = float(to_obj['latitude'])
             lon1 = float(to_obj['longitude'])
-            phases = tag['phases']
-            if tag['class'].startswith('underground'):
-                opacity = 0.3
-            else:
-                opacity = 0.7
             pos0 = (lat0,lon0)
             pos1 = (lat1,lon1)
             if pos0 != pos1:
-                obj = folium.PolyLine([pos0,pos1],color=color,weight=len(phases)*2,opacity=opacity,popup=popup,name=name)
+                if "phases" in tag.keys():
+                    weight = len(tag['phases'])*2
+                    if tag['class'].startswith('underground'):
+                        opacity = 0.3
+                    else:
+                        opacity = 0.7
+                else:
+                    opacity = 0.5
+                    weight = 3
+                    if "map_opacity" in tag.keys():
+                        opacity = tag["map_opacity"]
+                    if "map_weight" in tag.keys():
+                        weight = tag["map_weight"]
+                obj = folium.PolyLine([pos0,pos1],color=color,weight=weight,opacity=opacity,popup=popup,name=name)
             else:
                 obj = folium.Marker(pos,icon=icon,popup=popup,name=name)
-        except: # apparently not a link, so it's a node or other object
+        except Exception as err: # apparently not a link, so it's a node or other object
+            #print(f"DEBUG: {err}",file=sys.stderr)
             if not icon:
                 warning(f"object '{name}' has no known icon (class '{oclass})'")
                 icon = folium.Icon(color=color)
             if pos:
                 obj = folium.Marker(pos,icon=icon,popup=popup,name=name)
-        obj.add_to(cluster)
+        if obj:
+            obj.add_to(cluster)
     if mouseposition:
         folium.plugins.MousePosition(auto_start=True,position=mouseposition).add_to(map)
     if search:
@@ -189,6 +216,11 @@ def get_current_color(C,C0,C1):
         return current_colors['emergency']
 
 def get_color(tag):
+
+    # direct color assignment
+    if "map_color" in tag.keys():
+        return tag["map_color"]
+
     # try node scheme first
     try:
         VN = float(tag['nominal_voltage'].split()[0])