Update from @aliebman

cemo/cluster.py

@@ -42,12 +42,12 @@ def __init__(self,
                  firstdow=4,
                  lastdow=3,
                  max_d=12,
-                 regions=[1, 2, 3, 4, 5],
+                 regions=None,
                  maxsynth=False):
         self.firstdow = firstdow  # Day of week starting period
         self.lastdow = lastdow  # Day of week ending period`
         self.max_d = max_d  # Maximum number of clusters
-        self.regions = regions  # NEM region tuple
+        self.regions = range(1, 6) if regions is None else regions  # NEM region tuple
         self.maxsynth = maxsynth
 
         # make week pattern into a list
@@ -113,13 +113,13 @@ def _get_region_data(self, region):
 
         return X1
 
-    def _calc_Xsynth(self, max=False):
+    def _calc_Xsynth(self, max_p=False):
         # empty array of synthetic individual in each cluster
         self.Xsynth = np.empty((self.max_d, self.nplen))
 
         for k in range(self.max_d):
             # calculate synthetic individual per clusters
-            if max:
+            if max_p:
                 self.Xsynth[k] = self.Xclus[k].max(axis=0)[:self.nplen]
             else:
                 self.Xsynth[k] = self.Xclus[k].mean(axis=0)[:self.nplen]
@@ -140,7 +140,7 @@ def clusterset(self, max_d, method='average', metric='cityblock'):
             self.Xclus[self.cluster[j] - 1] \
                 = np.row_stack((self.Xclus[self.cluster[j] - 1], X2[j, :]))
         # genereate Xsynth (by default is the max of all features in cluster)
-        self._calc_Xsynth(max=self.maxsynth)
+        self._calc_Xsynth(max_p=self.maxsynth)
 
         # Obtain the date index for the observation in each cluster
         # closest to their respective cluster mean

cemo/const.py

@@ -186,7 +186,7 @@
     11: 20975.96,
     12: 10487.98,
     16: 52439.9,
-    19: 52439.9  # FIXME Sould this be 10k like for gas?
+    19: 52439.9  # FIXME Should this be 10k like for gas?
 }
 
 DEFAULT_TECH_LIFETIME = {
@@ -225,8 +225,7 @@
         17: 0,
     },
     5: {
-        11: 4000 + 3000 + 1000 +
-        29.9,  # Brkn hill + 1/2 of Murray R + 1/2 of Riverland + existing
+        11: 4000 + 3000 + 1000 + 29.9,  # Brkn hill + 1/2 of Murray R + 1/2 of Riverland + existing
         12: 1870 + 1620 + 232.5 + 0,
         17: 620 + 527.5 + 77.5 + 199,  #
     },

cemo/initialisers.py

@@ -13,6 +13,7 @@
 
 
 def init_year_correction_factor(model):
+    #pylint: disable=unused-argument
     '''Calculate factor to adjust dispatch periods different to 8760 hours'''
     ystr = model.t.last()
     year = int(ystr[:4])
@@ -23,81 +24,109 @@ def init_year_correction_factor(model):
 
 
 def init_zones_in_regions(model):
+    #pylint: disable=unused-argument
     '''Return zones in region tuples for declared regions'''
     for i in cemo.const.ZONES_IN_REGIONS:
         if i[0] in model.regions and i[1] in model.zones:
             yield i
 
 
 def init_region_intercons(model):
+    #pylint: disable=unused-argument
     '''Return regional interconnectors for declared regions'''
     for i in cemo.const.REGION_INTERCONS:
         if i[0] in model.regions and i[1] in model.regions:
             yield i
 
 
 def init_stor_rt_eff(model, tech):
+    #pylint: disable=unused-argument
     '''Default return efficiency for storage techs'''
     return cemo.const.DEFAULT_STOR_PROPS["rt_eff"].get(tech, 0)
 
 
 def init_stor_charge_hours(model, tech):
+    #pylint: disable=unused-argument
     '''Default charge hours for storage tech'''
     return cemo.const.DEFAULT_STOR_PROPS["charge_hours"].get(tech, 0)
 
 
 def init_hyb_col_mult(model, tech):
+    #pylint: disable=unused-argument
     '''Default collector multiple for hybrid tech'''
     return cemo.const.DEFAULT_HYB_PROPS["col_mult"].get(tech, 0)
 
 
 def init_hyb_charge_hours(model, tech):
+    #pylint: disable=unused-argument
     '''Default charge hours for hybrid tech'''
     return cemo.const.DEFAULT_HYB_PROPS["charge_hours"].get(tech, 0)
 
 
-def init_intercon_prop_factor(m, source, dest):
+def init_intercon_prop_factor(model, source, dest):
+    #pylint: disable=unused-argument
     '''Initialise interconnector proportioning factors'''
     return cemo.const.INTERCON_PROP_FACTOR.get(source).get(dest, 0)
 
 
-def init_intercon_trans_limit(m, source, dest):
+def init_intercon_trans_limit(model, source, dest):
+    #pylint: disable=unused-argument
+    '''Initialise interconecto transmission limits'''
     return cemo.const.INTERCON_TRANS_LIMIT.get(source).get(dest)
 
 
 def init_default_fuel_price(model, zone, tech):
+    #pylint: disable=unused-argument
+    '''Assign default price across zone and technologies'''
     return cemo.const.DEFAULT_FUEL_PRICE.get(tech, 100.0)
 
 
 def init_default_heat_rate(model, zone, tech):
+    #pylint: disable=unused-argument
+    '''Initialise default heat rate for fuel based generators in each zone'''
     return cemo.const.DEFAULT_HEAT_RATE.get(tech, 15.0)
 
 
 def init_default_fuel_emit_rate(model, tech):
+    #pylint: disable=unused-argument
+    '''Default fuel emission rate for fuel based generators'''
     return cemo.const.DEFAULT_FUEL_EMIT_RATE.get(tech, 800)
 
 
 def init_cost_retire(model, tech):
+    #pylint: disable=unused-argument
+    '''Default retirement/rehabilitation cost in $/MW per technology'''
     return cemo.const.DEFAULT_RETIREMENT_COST.get(tech, 60000.0)
 
 
 def init_default_lifetime(model, tech):
+    #pylint: disable=unused-argument
+    '''Default lifetime for technologies'''
     return cemo.const.DEFAULT_TECH_LIFETIME.get(tech, 30.0)
 
 
 def init_gen_build_limit(model, zone, tech):
+    #pylint: disable=unused-argument
+    ''' Default build limits per technology and per zone'''
     return cemo.const.DEFAULT_BUILD_LIMIT.get(zone).get(tech, 100000)
 
 
 def init_fcr(model, tech):
+    #pylint: disable=unused-argument
+    '''Calculate fixed charge rate for each technology'''
     return model.all_tech_discount_rate / (
         (model.all_tech_discount_rate + 1)**model.all_tech_lifetime[tech] -
         1) + model.all_tech_discount_rate
 
 
 def init_cap_factor(model, zone, tech, time):
+    #pylint: disable=unused-argument
+    '''Default capacity factor per hour per technology and per zone.
+        Note:Default to zero means technology does not generate'''
     return cemo.const.GEN_CAP_FACTOR.get(tech, 0)
 
 
 def init_max_hydro(model, zone):
+    #pylint: disable=unused-argument
+    '''Default maximum hydro generation per year in each zone'''
     return cemo.const.DEFAULT_HYDRO_MWH_MAX.get(zone, 0)

cemo/jsonify.py

@@ -138,10 +138,10 @@ def jsonify(inst):
 
 def jsoninit(inst):
     '''Produce JSON output sufficient to initialise a cemo model'''
-    input = jsonify(inst)
+    inp = jsonify(inst)
     out = {}
-    out.update(input['sets'])
-    out.update(input['params'])
+    out.update(inp['sets'])
+    out.update(inp['params'])
 
     del out['zones_per_region']
     del out['gen_tech_per_zone']

cemo/multi.py

@@ -49,7 +49,7 @@ def roundup(cap):
     Catching small negative numbers due to solver numerical tolerance.
     Let big negative numners pass to raise exception.
     '''
-    if cap > -1e-6 and cap < 0:
+    if -1e-6 < cap < 0:
         return 0
     return round(cap, 2)
 
@@ -119,12 +119,12 @@ def __init__(self, cfgfile, solver='cbc', log=False, tmpdir=tempfile.mkdtemp() +
             self.nem_re_disp_ratio = json.loads(Scenario['nem_re_disp_ratio'])
         # Keep track of policy options to configure model instances down the line
         self.model_options = {
-            'nem_ret_ratio': (True if self.nem_ret_ratio is not None else False),
-            'nem_ret_gwh': (True if self.nem_ret_gwh is not None else False),
-            'region_ret_ratio': (True if self.region_ret_ratio is not None else False),
-            'emitlimit': (True if self.emitlimit is not None else False),
-            'nem_disp_ratio': (True if self.nem_disp_ratio is not None else False),
-            'nem_re_disp_ratio': (True if self.nem_re_disp_ratio is not None else False),
+            'nem_ret_ratio': self.nem_ret_ratio is not None,
+            'nem_ret_gwh': self.nem_ret_gwh is not None,
+            'region_ret_ratio': self.region_ret_ratio is not None,
+            'emitlimit': self.emitlimit is not None,
+            'nem_disp_ratio': self.nem_disp_ratio is not None,
+            'nem_re_disp_ratio': self.nem_re_disp_ratio is not None
         }
 
         self.discountrate = Scenario['discountrate']
@@ -339,7 +339,7 @@ def exogenous_capacity(self, a):
 
     def tracetechs(self):  # TODO refactor this and how tech sets populate template
         self.fueltech = {}
-        self.committech ={}
+        self.committech = {}
         self.regentech = {}
         self.dispgentech = {}
         self.redispgentech = {}
@@ -496,7 +496,12 @@ def produce_exogenous_capacity(self, year):
                     exogenous_capacity += '#Exogenous capacity entry ' + key + '\n'
                     exogenous_capacity += 'param ' + key + ':=\n'
                     exogenous_capacity += cap[['zone', 'tech', 'value']
-                                              ].to_string(header=False, index=False)
+                                              ].to_string(header=False, index=False,
+                                                          formatters={
+                                                              'zone': lambda x: '%i' % x,
+                                                              'tech': lambda x: '%i' % x,
+                                                              'value': lambda x: '%10.2f' % x,
+                                                          })
                     exogenous_capacity += '\n;\n'
 
         return exogenous_capacity

cemo/rules.py

@@ -338,16 +338,15 @@ def con_opcap(model, z, n):  # z and n come both from TechinZones
         return model.gen_cap_op[z, n] == model.gen_cap_initial[z, n] \
             + (model.gen_cap_exo[z, n] - model.gen_cap_exo_neg[z, n])
 
-    else:
-        if n in model.retire_gen_tech:
-            return model.gen_cap_op[z, n] == model.gen_cap_initial[z, n] \
-                + (model.gen_cap_exo[z, n] - model.gen_cap_exo_neg[z, n])\
-                + model.gen_cap_new[z, n]\
-                - model.gen_cap_ret[z, n] - \
-                (model.ret_gen_cap_exo[z, n] - model.gen_cap_ret_neg[z, n])
-        return model.gen_cap_op[z, n] == model.gen_cap_initial[z, n]\
+    if n in model.retire_gen_tech:
+        return model.gen_cap_op[z, n] == model.gen_cap_initial[z, n] \
             + (model.gen_cap_exo[z, n] - model.gen_cap_exo_neg[z, n])\
-            + model.gen_cap_new[z, n]
+            + model.gen_cap_new[z, n]\
+            - model.gen_cap_ret[z, n] - \
+            (model.ret_gen_cap_exo[z, n] - model.gen_cap_ret_neg[z, n])
+    return model.gen_cap_op[z, n] == model.gen_cap_initial[z, n]\
+        + (model.gen_cap_exo[z, n] - model.gen_cap_exo_neg[z, n])\
+        + model.gen_cap_new[z, n]
 
 
 def con_stcap(model, z, s):  # z and n come both from TechinZones

cemo/utils.py

@@ -48,7 +48,7 @@ def _techsinregion(instance, region):  # pragma: no cover
         techsinregion = techsinregion | instance.gen_tech_per_zone[z]()
         techsinregion = techsinregion | instance.hyb_tech_per_zone[z]()
         techsinregion = techsinregion | instance.stor_tech_per_zone[z]()
-    return sorted(techsinregion, key=lambda x: cemo.const.DISPLAY_ORDER.index(x))
+    return sorted(techsinregion, key=cemo.const.DISPLAY_ORDER.index)
 
 
 def palette(instance, techsinregion):  # pragma: no cover
@@ -211,7 +211,7 @@ def _printemissionrate(instance):
     emrate = sum(value(cemo.rules.emissions(instance, r))
                  for r in instance.regions) /\
         (sum(value(cemo.rules.dispatch(instance, r)) for r in instance.regions)+1.0e-12)
-    print("Total Emission rate: %s kg/MWh" % str(emrate))
+    print("Total Emission rate: %6.3f kg/MWh" % emrate)
 
 
 def _printunserved(instance):
@@ -281,11 +281,11 @@ def printstats(instance):
     print("End of results for %s" % instance.name, flush=True)
 
 
-def plotcluster(cluster, row=3, col=4, ylim=[5500, 16000]):  # pragma: no cover
+def plotcluster(cluster, row=3, col=4, ylim=None):  # pragma: no cover
     # Plot cluster result from full set of weeks, cluster weeks and weights
     t = range(1, cluster.nplen + 1)
     # Make  row * col subplots
-    f, axarr = plt.subplots(row, col, sharex=True)
+    axarr = plt.subplots(row, col, sharex=True)[1]
     # Plot each observation in their respective cluster plot
     for i in range(cluster.periods):
         axarr.flat[cluster.cluster[i] -
@@ -302,6 +302,10 @@ def plotcluster(cluster, row=3, col=4, ylim=[5500, 16000]):  # pragma: no cover
                            marker='+')  # closest observation
     # make yrange the same in all plots
     for ax in axarr.flat:
-        ax.set_ylim(ylim[0], ylim[1])
+        if ylim is None:
+            # default
+            ax.set_ylim(5500, 16000)
+        else:
+            ax.set_ylim(ylim[0], ylim[1])
     # Show results
     plt.show(figsize=(14, 9))

msolve.py

@@ -18,7 +18,7 @@
 
 
 def valid_file(param):
-    base, ext = os.path.splitext(param)
+    ext = os.path.splitext(param)[1]
     if ext not in ('.cfg'):
         raise argparse.ArgumentTypeError('File must have a cfg extension')
     return param
@@ -35,7 +35,7 @@ def valid_file(param):
     "config",
     help="Specify a configuration file for simulation" +
     " Note: Python configuration files named CONFIG.cfg",
-    type=lambda f: valid_file(f),
+    type=valid_file,
     metavar='CONFIG')
 # Obtain a solver name from command line, default cbc
 parser.add_argument(
@@ -55,8 +55,7 @@ def valid_file(param):
 parser.add_argument(
     "-k",
     "--keepfiles",
-    help=
-    "Save generated files onto a folder with the same name as the configuration file",
+    help="Save generated files onto a folder with the same name as the configuration file",
     action='store_true')
 
 # parse arguments into args structure

requirements.txt

@@ -0,0 +1,23 @@
+appdirs==1.4.3
+atomicwrites==1.3.0
+attrs==19.1.0
+cycler==0.10.0
+kiwisolver==1.0.1
+matplotlib==3.0.3
+more-itertools==7.0.0
+nose==1.3.7
+numpy==1.16.2
+pandas==0.24.2
+pluggy==0.9.0
+ply==3.11
+py==1.8.0
+PyMySQL==0.9.3
+Pyomo==5.6.1
+pyparsing==2.3.1
+pytest==4.4.0
+python-dateutil==2.8.0
+pytz==2018.9
+PyUtilib==5.6.5
+scipy==1.2.1
+si-prefix==1.2.2
+six==1.12.0

ssolve.py

@@ -95,11 +95,7 @@ def check_arg(config_file, parameter):
                      region_ret_ratio=check_arg(MODEL_NAME, 'region_ret_ratio'),
                      nem_re_disp_ratio=check_arg(MODEL_NAME, 'nem_re_disp_ratio'))
 # create a specific instance using file modelName.dat
-try:
-    INSTANCE = MODEL.create_instance(MODEL_NAME + '.dat')
-except Exception as ex:
-    print("openCEM solve.py: ", ex)
-    sys.exit(1)  # exit gracefully if file does not exist
+INSTANCE = MODEL.create_instance(MODEL_NAME + '.dat')
 
 # Produce only a debugging printout of model and then exit
 if ARGS.printonly:

tests/Sim2020.dat

@@ -690,7 +690,7 @@ order BY regions,t;": [regions,t] region_net_demand;
 #Custom cost entry for cost_stor_build
 param cost_stor_build:=
 13 14  999999.00
-5 21 2000000.00
+ 5 21 2000000.00
 ;
 #Custom cost entry for cost_gen_fom
 param cost_gen_fom:=