Add MECS2014 data from IEA (#773)

* Add MECS2014 data from IEA

* Add config.py

* Fix documentation typo

* Remove invalid data

* Merge MECS and NERC data sets

* Move data files to data folder

* Update NAICS Data.md

* Add support for industrial capacity load variation

* Change invalid NAICS record from error to warning

* Fix handling of NAICS descriptions

* Update naics_data_file.csv

* Remove duplicate NAICS codes

* Create test_industrial_variable.glm

docs/Module/Industrial/Industrial.md

@@ -10,7 +10,7 @@ object industrial
 	naics_code <naics-code>;
 	total_power <power>;
 	power_pf <power-factor>;
-	impedance_pt <power-factor>
+	impedance_pf <power-factor>
 }
 ~~~
 

docs/Module/Industrial/NAICS Data.md

@@ -15,7 +15,7 @@ module industrial
 
 The NAICS data file contains the load composition for various industrial loads. All motor and power electronic loads are considered to be constant power.
 
-The current data supports the following facilities:
+The current data supports the following NERC-defined facilities:
 
 | NAICS Code | Industrial Load Type       | NERC Code | Motor A | Motor B | Motor C | Motor D | Power Electronics | Constant Impedance | Constant Current |
 | ---------- | -------------------------- | --------- | ------- | ------- | ------- | ------- | ----------------- | ------------------ | ---------------- |
@@ -38,6 +38,8 @@ The current data supports the following facilities:
 | 482        | Transportation - Rail      | IND_RAIL  | 0       | 0       | 0.05    | 0       | 0.95              | 0                  | 0                |
 | 518        | Server Farm                | IND_SRF   | 0       | 0       | 0.1     | 0       | 0.9               | 0                  | 0                |
 
+In addition load compositions from the IEA MECS 2014 data are included. See [EIA 2014 MECS Survey Data](https://www.eia.gov/consumption/manufacturing/data/2014/) Tables 5.1, 9.1, and 11.1 for details.
+
 # See also
 
 * [[/Module/Industrial]]

industrial/autotest/test_industrial_variable.glm

@@ -0,0 +1,47 @@
+// test_industial.glm
+// Copyright (C) 2020, Regents of the Leland Stanford Junior University
+
+#set savefile=test_industrial.json
+
+clock
+{
+	timezone "US/CA/San Francisco";
+	starttime "2020-01-01 00:00:00 PST";
+	stoptime "2021-01-01 00:00:00 PST";
+}
+
+module tape;
+module powerflow;
+
+object load
+{
+	phases ABCD;
+	nominal_voltage 12.47 kV;
+	name "test";
+	object recorder 
+	{
+		file "test_industrial_variable.csv";
+		property constant_power_A,constant_power_B,constant_power_C,constant_current_A,constant_current_B,constant_current_C,constant_impedance_A,constant_impedance_B,constant_impedance_C;
+		interval 1h;
+	};
+}
+
+module industrial
+{
+	naics_data_file "naics_data_file.csv";
+}
+
+schedule operations
+{
+	* 6-18 * * 1-5 1.0;
+	* 19-5 * * 1-5 0.5;
+	* * * * 6-0 0.2;
+}
+
+object industrial
+{
+	parent "test";
+	naics_code 3254; // pharmaceutical plant
+	operating_capacity 1.0 MW;
+	operating_factor operations*1.0;
+}

industrial/data/config.py

@@ -0,0 +1,18 @@
+enduses = {
+	"Conventional Boiler Use" : {"B":0.1,"Z":0.9}, # heaters and pumps
+	"CHP and/or Cogeneration Process" : {"B":0.5,"C":0.5}, # pumps and fans
+	"Process Heating" : {"B":0.1,"Z":0.9}, # heaters and pumps
+	"Process Cooling and Refrigeration" : {"A":0.8,"B":0.1,"C":0.1}, # compressors, pumps, and fans
+	"Machine Drive" : {"B":1.0}, # machine drives
+	"Electro-Chemical Processes" : {"A":0.4,"B":0.05,"C":0.05,"Z":0.5}, # compressors, pumps, fans, and heaters
+	"Other Process Use" : {"B":1.0},
+	"Facility HVAC (g)" : {"A":0.4,"B":0.05,"C":0.05,"Z":0.5},
+	"Facility Lighting" : {"E":0.8,"I":0.2},
+	"Other Facility Support" : {"E":1.0},
+	"Onsite Transportation" : {"B":1.0},
+	"Other Nonprocess Use" : {"D":0.1,"Z":0.8,"I":0.1},
+	"End Use Not Reported" : {"D":0.1,"Z":0.8,"I":0.1},
+}
+mecs_csv = "mecs_data.csv"
+nerc_csv = "nerc_data.csv"
+naics_csv = "../naics_data_file.csv"

industrial/data/mecs.py

@@ -0,0 +1,168 @@
+"""Create NAICS data file from MECS and NERC data files
+
+Run command:
+
+	host% python3 mecs.py
+	Loaded table 9.1 ok
+	Loaded table 11.1 ok
+	Loaded table 5.1 ok
+	Saved ../mecs_data.csv ok
+	Loaded ../nerc_data.csv ok
+	Merged ok
+	Saved ../naics_data_file.csv ok
+	
+"""
+from openpyxl import load_workbook
+from pandas import DataFrame, concat, read_csv
+
+import config
+
+table5 = load_workbook(filename="MECS2014_Table_5-1.xlsx")["Table 5.1"]
+table9 = load_workbook(filename="MECS2014_Table_9-1.xlsx")["Table 9.1"]
+table11 = load_workbook(filename="MECS2014_Table_11-1.xlsx")["Table 11.1"]
+
+columns = [
+	"NAICS", "DESCRIPTION", "REGION", # index
+	"FLOORAREA","SITES","FLOORAREA_PER_SITE","BUILDINGS","BUILDINGS_PER_SITE", # Table 9
+	"PURCHASES","TRANSFERS","GENERATION","SALES","NET_DEMAND", # Table 11
+	]
+
+#
+# Table 9 - Buildings
+#
+data = []
+for values in table9.iter_rows(min_row=18,min_col=1,max_row=98,max_col=7,values_only=True):
+	row = [int(values[0])]
+	row.extend(values[1:])
+	data.append(row)
+mecs9 = DataFrame.from_records(data,index=["NAICS"],
+			columns = ["NAICS","DESCRIPTION","FLOORAREA[Msf]","SITES[unit]","FLOORAREA_PER_SITE[sf/unit]","BUILDINGS[unit]","BUILDINGS_PER_SITE[unit/unit]"])
+valid_codes = mecs9.index
+print("Loaded table 9.1 ok")
+
+#
+# Table 11 - Regional electricity use
+#
+regions = {
+	"NERC":(13,93),
+	"NORTHEAST":(97,177),
+	"MIDWEST":(181,261),
+	"SOUTH":(265,345),
+	"WEST":(349,429),
+	}
+data = []
+for region, rows in regions.items():
+		for values in table11.iter_rows(min_row=rows[0],min_col=1,max_row=rows[1],max_col=7,values_only=True):
+			try:
+				index = int(values[0].strip())
+				if index in valid_codes:
+					row = [index,region]
+					for value in values[2:]:
+						if value == "W": # withheld due to low sample count
+							value = float('nan')
+						elif value == "Q": # withheld due to high RSE
+							value = float('nan')
+						elif value == "*": # value < 0.5
+							value = 0.0
+						elif value == "NA": # value not available
+							value = float('nan')
+						row.append(value)
+					data.append(row)
+			except:
+				pass
+mecs11 = DataFrame.from_records(data,index=["NAICS"],
+			columns=["NAICS","REGION","PURCHASES[GWh]","TRANSFERS[GWh]","GENERATION[GWh]","SALES[GWh]","NET_DEMAND[GWh]"])
+print("Loaded table 11.1 ok")
+
+#
+# Table 5 - End-use decomposition
+#
+data = []
+index = None
+electrification = [None,None,None,None]
+for values in table5.iter_rows(min_row=17,min_col=1,max_row=1819,max_col=4,values_only=True):
+	if values[0]:
+		try:
+			if index:
+				row = [index]
+				row.extend(electrification)
+				for enduse in ["A","B","C","D","E","I","Z"]:
+					row.append(enduse_composition[enduse]/enduse_composition["TOTAL"])
+				data.append(row)
+			index = int(values[0])
+			if not index in valid_codes:
+				index = None
+		except:
+			continue
+		enduse_composition = {"A":0.0,"B":0.0,"C":0.0,"D":0.0,"E":0.0,"I":0.0,"Z":0.0,"TOTAL":0.0}
+	elif not values[1]:
+		continue
+	tag = values[1].strip()
+	if index and values[2] and values[3]:
+		try:
+			electric = float(values[3])
+			if tag == "TOTAL FUEL CONSUMPTION":
+				total = float(values[2])
+				electrification[0] = electric
+				electrification[1] = total/electric*3.412
+			elif tag == "End Use Not Reported":
+				total = float(values[2])
+				electrification[2] = electric
+				electrification[3] = total/electric*3.412
+				components = config.enduses[tag]
+			elif tag in config.enduses.keys():
+				components = config.enduses[tag]
+			if components:
+				for enduse, value in components.items():
+					enduse_composition[enduse] += value
+					enduse_composition["TOTAL"] += value
+				components = None				
+		except:
+			pass
+if index:
+	row = [index]
+	row.extend(electrification)
+	data.append(row)
+mecs5 = DataFrame.from_records(data,index=["NAICS"],
+			columns=["NAICS","REPORTED_ELECTRIC[GWh]","REPORTED_ELECTRIFICATION[pu]","UNREPORTED_ELECTRIC[GWh]","UNREPORTED_ELECTRIFICATION[pu]",
+			"A","B","C","D","E","I","Z"])
+print("Loaded table 5.1 ok")
+
+mecs = mecs9.join(mecs11).join(mecs5)
+local = mecs["REGION"]!="NERC"
+for column in ["FLOORAREA[Msf]","BUILDINGS[unit]","SITES[unit]"]:
+	mecs.at[local,column] = float('nan')
+
+mecs.index.name = "NAICS"
+mecs.reset_index(inplace=True)
+mecs.set_index(["NAICS","REGION"],inplace=True)
+
+mecs.to_csv(config.mecs_csv)
+print(f"Saved {config.mecs_csv} ok")
+
+nerc = read_csv(config.nerc_csv,converters={"NAICS":int}).set_index("Code")
+print(f"Loaded {config.nerc_csv} ok")
+for row in mecs.iterrows():
+	name = f"MECS_{row[0][0]}"
+	if row[0][0] in list(map(lambda x:int(x),list(nerc["NAICS"]))):
+		continue # do not create a duplicate entry
+	nerc.loc[name,"NAICS"] = int(row[0][0])
+	for item, value in dict(
+			DESCRIPTION="Industrial Load Type",
+			A="IA",
+			B="IB",
+			C="IC",
+			D="MD",
+			E="PwrEl",
+			I="I",
+			Z="Z").items():
+		if type(row[1][item]) is float:
+			nerc.loc[name,value] = round(row[1][item],3)
+		elif type(row[1][item]) is str and len(row[1][item]) > 0:
+			nerc.loc[name,value] = row[1][item].replace(',','').strip()
+		else:
+			nerc.loc[name,value] = "NA"
+print(f"Merged ok")
+
+nerc.reset_index().set_index(["NAICS","Code"]).to_csv(config.naics_csv,float_format="%g")
+print(f"Saved {config.naics_csv} ok")