From cd37b4a1b6b7f5d4240a5ed2a1ad242afece53ae Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Fri, 10 Mar 2017 15:18:41 +1030 Subject: [PATCH 01/10] updated __init__.py to include missing encoders --- src/nupic/encoders/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nupic/encoders/__init__.py b/src/nupic/encoders/__init__.py index 370c977dd8..32d7250545 100644 --- a/src/nupic/encoders/__init__.py +++ b/src/nupic/encoders/__init__.py @@ -20,6 +20,7 @@ # ---------------------------------------------------------------------- from scalar import ScalarEncoder +from random_distributed_scalar import RandomDistributedScalarEncoder from adaptivescalar import AdaptiveScalarEncoder from date import DateEncoder from logenc import LogEncoder @@ -29,7 +30,8 @@ from scalarspace import ScalarSpaceEncoder from coordinate import CoordinateEncoder from geospatial_coordinate import GeospatialCoordinateEncoder -from nupic.encoders.pass_through_encoder import PassThroughEncoder +from pass_through_encoder import PassThroughEncoder +from sparse_pass_through_encoder import SparsePassThroughEncoder # multiencoder must be imported last because it imports * from this module! from multi import MultiEncoder from utils import bitsToString From 7a1c8b51a35e5545dcb6c515122c3792c4797b20 Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Mon, 13 Mar 2017 11:26:32 +1030 Subject: [PATCH 02/10] updated tests and refactored multi.py --- src/nupic/encoders/multi.py | 30 +++++++++---------- .../encoders/pass_through_encoder_test.py | 2 +- .../random_distributed_scalar_test.py | 4 +-- .../sparse_pass_through_encoder_test.py | 2 +- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/nupic/encoders/multi.py b/src/nupic/encoders/multi.py index 069e220272..469f53ff6c 100644 --- a/src/nupic/encoders/multi.py +++ b/src/nupic/encoders/multi.py @@ -19,22 +19,22 @@ # http://numenta.org/licenses/ # ---------------------------------------------------------------------- -from nupic.encoders.base import Encoder -from nupic.encoders.scalar import ScalarEncoder -from nupic.encoders.adaptivescalar import AdaptiveScalarEncoder -from nupic.encoders.date import DateEncoder -from nupic.encoders.logenc import LogEncoder -from nupic.encoders.category import CategoryEncoder -from nupic.encoders.sdrcategory import SDRCategoryEncoder -from nupic.encoders.delta import DeltaEncoder -from nupic.encoders.scalarspace import ScalarSpaceEncoder -from nupic.encoders.pass_through_encoder import PassThroughEncoder -from nupic.encoders.sparse_pass_through_encoder import SparsePassThroughEncoder -from nupic.encoders.coordinate import CoordinateEncoder -from nupic.encoders.geospatial_coordinate import GeospatialCoordinateEncoder +from nupic.encoders import Encoder +from nupic.encoders import ScalarEncoder +from nupic.encoders import AdaptiveScalarEncoder +from nupic.encoders import DateEncoder +from nupic.encoders import LogEncoder +from nupic.encoders import CategoryEncoder +from nupic.encoders import SDRCategoryEncoder +from nupic.encoders import DeltaEncoder +from nupic.encoders import ScalarSpaceEncoder +from nupic.encoders import PassThroughEncoder +from nupic.encoders import SparsePassThroughEncoder +from nupic.encoders import CoordinateEncoder +from nupic.encoders import GeospatialCoordinateEncoder # multiencoder must be imported last because it imports * from this module! -from nupic.encoders.utils import bitsToString -from nupic.encoders.random_distributed_scalar import RandomDistributedScalarEncoder +from nupic.encoders import bitsToString +from nupic.encoders import RandomDistributedScalarEncoder diff --git a/tests/unit/nupic/encoders/pass_through_encoder_test.py b/tests/unit/nupic/encoders/pass_through_encoder_test.py index b12a247c36..c0592f49e6 100755 --- a/tests/unit/nupic/encoders/pass_through_encoder_test.py +++ b/tests/unit/nupic/encoders/pass_through_encoder_test.py @@ -28,7 +28,7 @@ import numpy -from nupic.encoders.pass_through_encoder import PassThroughEncoder +from nupic.encoders import PassThroughEncoder try: import capnp diff --git a/tests/unit/nupic/encoders/random_distributed_scalar_test.py b/tests/unit/nupic/encoders/random_distributed_scalar_test.py index 4c3315ac15..948bd2ced0 100755 --- a/tests/unit/nupic/encoders/random_distributed_scalar_test.py +++ b/tests/unit/nupic/encoders/random_distributed_scalar_test.py @@ -29,9 +29,7 @@ from nupic.data import SENTINEL_VALUE_FOR_MISSING_DATA from nupic.data.fieldmeta import FieldMetaType from nupic.support.unittesthelpers.algorithm_test_helpers import getSeed -from nupic.encoders.random_distributed_scalar import ( - RandomDistributedScalarEncoder -) +from nupic.encoders import RandomDistributedScalarEncoder try: import capnp diff --git a/tests/unit/nupic/encoders/sparse_pass_through_encoder_test.py b/tests/unit/nupic/encoders/sparse_pass_through_encoder_test.py index 7f9a5685c3..6206a5a6d5 100755 --- a/tests/unit/nupic/encoders/sparse_pass_through_encoder_test.py +++ b/tests/unit/nupic/encoders/sparse_pass_through_encoder_test.py @@ -28,7 +28,7 @@ import numpy -from nupic.encoders.sparse_pass_through_encoder import SparsePassThroughEncoder +from nupic.encoders import SparsePassThroughEncoder try: From 137b966c891629594535159c2d7cec6ac3512375 Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Mon, 13 Mar 2017 14:33:53 +1030 Subject: [PATCH 03/10] fixed base encoder import --- src/nupic/encoders/multi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nupic/encoders/multi.py b/src/nupic/encoders/multi.py index 469f53ff6c..3ee4183d7b 100644 --- a/src/nupic/encoders/multi.py +++ b/src/nupic/encoders/multi.py @@ -19,7 +19,7 @@ # http://numenta.org/licenses/ # ---------------------------------------------------------------------- -from nupic.encoders import Encoder +from nupic.encoders.base import Encoder from nupic.encoders import ScalarEncoder from nupic.encoders import AdaptiveScalarEncoder from nupic.encoders import DateEncoder From 69a66544cbffe6838e9364f753a454bd8e5db3ce Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Fri, 19 May 2017 12:02:09 +0930 Subject: [PATCH 04/10] yep --- src/nupic/encoders/multi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nupic/encoders/multi.py b/src/nupic/encoders/multi.py index 3ee4183d7b..a5ee82cc6a 100644 --- a/src/nupic/encoders/multi.py +++ b/src/nupic/encoders/multi.py @@ -33,7 +33,7 @@ from nupic.encoders import CoordinateEncoder from nupic.encoders import GeospatialCoordinateEncoder # multiencoder must be imported last because it imports * from this module! -from nupic.encoders import bitsToString +from nupic.encoders.utils import bitsToString from nupic.encoders import RandomDistributedScalarEncoder From b03ebe14590e3d59c6b8f7086e31c3a97af5c73a Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Fri, 19 May 2017 12:57:14 +0930 Subject: [PATCH 05/10] Fixed encoder imports after name changes --- src/nupic/encoders/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nupic/encoders/__init__.py b/src/nupic/encoders/__init__.py index 32d7250545..48a1ca13ab 100644 --- a/src/nupic/encoders/__init__.py +++ b/src/nupic/encoders/__init__.py @@ -21,17 +21,17 @@ from scalar import ScalarEncoder from random_distributed_scalar import RandomDistributedScalarEncoder -from adaptivescalar import AdaptiveScalarEncoder +from adaptive_scalar import AdaptiveScalarEncoder from date import DateEncoder -from logenc import LogEncoder +from logarithm import LogEncoder from category import CategoryEncoder -from sdrcategory import SDRCategoryEncoder +from sdr_category import SDRCategoryEncoder from delta import DeltaEncoder -from scalarspace import ScalarSpaceEncoder +from scalar_space import ScalarSpaceEncoder from coordinate import CoordinateEncoder from geospatial_coordinate import GeospatialCoordinateEncoder -from pass_through_encoder import PassThroughEncoder -from sparse_pass_through_encoder import SparsePassThroughEncoder +from pass_through import PassThroughEncoder +from sparse_pass_through import SparsePassThroughEncoder # multiencoder must be imported last because it imports * from this module! from multi import MultiEncoder from utils import bitsToString From bcf4dd1083d706f09ce11ffb14980d64e63f6efc Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Mon, 22 May 2017 14:36:27 +0930 Subject: [PATCH 06/10] Fixed encoder import styling. --- src/nupic/encoders/geospatial_coordinate.py | 2 +- src/nupic/encoders/logarithm.py | 2 +- src/nupic/encoders/multi.py | 34 +++++++++------------ src/nupic/encoders/scalar_space.py | 4 +-- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/nupic/encoders/geospatial_coordinate.py b/src/nupic/encoders/geospatial_coordinate.py index 79dec27654..baf9e06ba9 100644 --- a/src/nupic/encoders/geospatial_coordinate.py +++ b/src/nupic/encoders/geospatial_coordinate.py @@ -23,7 +23,7 @@ import numpy from pyproj import Proj, transform -from nupic.encoders.coordinate import CoordinateEncoder +from nupic.encoders import CoordinateEncoder diff --git a/src/nupic/encoders/logarithm.py b/src/nupic/encoders/logarithm.py index 5e224a1055..c73eba22a7 100644 --- a/src/nupic/encoders/logarithm.py +++ b/src/nupic/encoders/logarithm.py @@ -25,7 +25,7 @@ from nupic.data import SENTINEL_VALUE_FOR_MISSING_DATA from nupic.data.field_meta import FieldMetaType from nupic.encoders.base import Encoder, EncoderResult -from nupic.encoders.scalar import ScalarEncoder +from nupic.encoders import ScalarEncoder class LogEncoder(Encoder): diff --git a/src/nupic/encoders/multi.py b/src/nupic/encoders/multi.py index a16044a53d..2e891fbd66 100644 --- a/src/nupic/encoders/multi.py +++ b/src/nupic/encoders/multi.py @@ -20,22 +20,18 @@ # ---------------------------------------------------------------------- from nupic.encoders.base import Encoder -from nupic.encoders import ScalarEncoder -from nupic.encoders import AdaptiveScalarEncoder -from nupic.encoders import DateEncoder -from nupic.encoders import LogEncoder -from nupic.encoders import CategoryEncoder -from nupic.encoders import SDRCategoryEncoder -from nupic.encoders import DeltaEncoder -from nupic.encoders import ScalarSpaceEncoder -from nupic.encoders import PassThroughEncoder -from nupic.encoders import SparsePassThroughEncoder -from nupic.encoders import CoordinateEncoder -from nupic.encoders import GeospatialCoordinateEncoder -# multiencoder must be imported last because it imports * from this module! -from nupic.encoders.utils import bitsToString -from nupic.encoders import RandomDistributedScalarEncoder - +from nupic.encoders import (ScalarEncoder, + AdaptiveScalarEncoder, + DateEncoder,LogEncoder, + CategoryEncoder, + SDRCategoryEncoder, + DeltaEncoder, + ScalarSpaceEncoder, + PassThroughEncoder, + SparsePassThroughEncoder, + CoordinateEncoder, + GeospatialCoordinateEncoder, + RandomDistributedScalarEncoder) # Map class to Cap'n Proto schema union attribute @@ -50,13 +46,13 @@ DeltaEncoder: "deltaEncoder", PassThroughEncoder: "passThroughEncoder", SparsePassThroughEncoder: "sparsePassThroughEncoder", + GeospatialCoordinateEncoder: "geospatialCoordinateEncoder", + ScalarSpaceEncoder: "scalarSpaceEncoder", RandomDistributedScalarEncoder: "randomDistributedScalarEncoder" } # Invert for fast lookup in MultiEncoder.read() -_ATTR_CLASS_MAP = {value:key for key, value in _CLASS_ATTR_MAP.items()} - - +_ATTR_CLASS_MAP = {value: key for key, value in _CLASS_ATTR_MAP.items()} class MultiEncoder(Encoder): diff --git a/src/nupic/encoders/scalar_space.py b/src/nupic/encoders/scalar_space.py index 5790f66ee9..5f4493a7a7 100644 --- a/src/nupic/encoders/scalar_space.py +++ b/src/nupic/encoders/scalar_space.py @@ -19,8 +19,8 @@ # http://numenta.org/licenses/ # ---------------------------------------------------------------------- from nupic.encoders.base import Encoder -from nupic.encoders.delta import DeltaEncoder -from nupic.encoders.adaptive_scalar import AdaptiveScalarEncoder +from nupic.encoders import (DeltaEncoder, + AdaptiveScalarEncoder) class ScalarSpaceEncoder(Encoder): From 6aff6ebd358fa9b008f0f75b0c663079a8fef463 Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Tue, 27 Feb 2018 13:31:23 +1030 Subject: [PATCH 07/10] Fixes #3803 added logging to run_swarm.py --- scripts/run_swarm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/run_swarm.py b/scripts/run_swarm.py index 9e6178da80..938bcf862b 100755 --- a/scripts/run_swarm.py +++ b/scripts/run_swarm.py @@ -25,6 +25,8 @@ import sys import os import optparse +import logging +logging.basicConfig() from nupic.swarming import permutations_runner from nupic.swarming.permutations_runner import DEFAULT_OPTIONS From bec483a1029a7fbb2b2fc4ffde121bf2b7f444ad Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Wed, 7 Mar 2018 10:21:51 +1030 Subject: [PATCH 08/10] fixed typo and metric schema for experiment --- .../swarming/exp_generator/experimentDescriptionSchema.json | 2 +- src/nupic/swarming/exp_generator/experiment_generator.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nupic/swarming/exp_generator/experimentDescriptionSchema.json b/src/nupic/swarming/exp_generator/experimentDescriptionSchema.json index 5956aedace..a8aa7742b4 100644 --- a/src/nupic/swarming/exp_generator/experimentDescriptionSchema.json +++ b/src/nupic/swarming/exp_generator/experimentDescriptionSchema.json @@ -113,7 +113,7 @@ "description": "inferenceArgs -- arguments for the type of inference you want to use" }, "prediction": { - "description": " [DEPRECATED] Prediction optimization options; at least one MUST be enabled. NOTE: at this time, non-temporal and temporalpredictions are mutually-exclusive (RunPermutations cannot yetoptimize for both). This is deprecated. Used inferenceType instead", + "description": " [DEPRECATED] Prediction optimization options; at least one MUST be enabled. NOTE: at this time, non-temporal and temporalpredictions are mutually-exclusive (RunPermutations cannot yet optimize for both). This is deprecated. Used inferenceType instead", "default": { "temporal": { "optimize": true diff --git a/src/nupic/swarming/exp_generator/experiment_generator.py b/src/nupic/swarming/exp_generator/experiment_generator.py index 0c9adf0c05..bb394dc1a8 100755 --- a/src/nupic/swarming/exp_generator/experiment_generator.py +++ b/src/nupic/swarming/exp_generator/experiment_generator.py @@ -1859,6 +1859,8 @@ def _generateMetricSpecs(options): def _generateExtraMetricSpecs(options): """Generates the non-default metrics specified by the expGenerator params """ global _metricSpecSchema + _metricSpecSchema = {} + _metricSpecSchema['properties'] = {} results = [] for metric in options['metrics']: From f57272867061189a1c44aa8ec5d3371e3e54fe6d Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Wed, 7 Mar 2018 12:30:25 +1030 Subject: [PATCH 09/10] Fixed typos in experimentDescriptionSchema.json Added WARNING message when permutations runner job fails Added nupic-site.xml to .gitignore --- .gitignore | 1 + .../exp_generator/experimentDescriptionSchema.json | 14 +++++++------- src/nupic/swarming/permutations_runner.py | 10 +++++++--- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index c50152d2b3..ab4724b681 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,4 @@ nupic.egg-info/ nupic/*py_region.* temp/ docs/_build_html +/src/nupic/support/nupic-site.xml diff --git a/src/nupic/swarming/exp_generator/experimentDescriptionSchema.json b/src/nupic/swarming/exp_generator/experimentDescriptionSchema.json index a8aa7742b4..e52811ab56 100644 --- a/src/nupic/swarming/exp_generator/experimentDescriptionSchema.json +++ b/src/nupic/swarming/exp_generator/experimentDescriptionSchema.json @@ -54,7 +54,7 @@ "integer", "null" ], - "description": "Maximum number of models to evaluate. This replaces the older location of this specification from the job params. " + "description": "Maximum number of models to evaluate. This replaces the older location of this specification from the job params." }, "spPermuteDecrement": { "default": true, @@ -113,7 +113,7 @@ "description": "inferenceArgs -- arguments for the type of inference you want to use" }, "prediction": { - "description": " [DEPRECATED] Prediction optimization options; at least one MUST be enabled. NOTE: at this time, non-temporal and temporalpredictions are mutually-exclusive (RunPermutations cannot yet optimize for both). This is deprecated. Used inferenceType instead", + "description": " [DEPRECATED] Prediction optimization options; at least one MUST be enabled. NOTE: at this time, non-temporal and temporal predictions are mutually-exclusive (RunPermutations cannot yet optimize for both). This is deprecated. Used inferenceType instead", "default": { "temporal": { "optimize": true @@ -177,7 +177,7 @@ "customErrorMetric": { "required": false, "type": "object", - "description": "Custom User definied Error metric that will be computed" + "description": "Custom User defined Error metric that will be computed" }, "spCoincInputPoolPct": { "default": 0.8, @@ -216,7 +216,7 @@ "description": "Flag to run baseline error metrics with the job for comparison" }, "metrics": { - "description": "Additional metrics to be generated, along with thedefault ones for the given inferenceType. Note: The specified metric should be compatible withe the giveninference", + "description": "Additional metrics to be generated, along with the default ones for the given inferenceType. Note: The specified metric should be compatible withe the given inference", "default": [], "items": { "required": true, @@ -267,7 +267,7 @@ "minLength": 1, "required": false, "type": "string", - "description": "[DEPRECATED] Name of field we're trying to predict. This is deprected. Used inferenceArgs.predictedField instead" + "description": "[DEPRECATED] Name of field we're trying to predict. This is deprecated. Used inferenceArgs.predictedField instead" }, "streamDef": { "minLength": 1, @@ -380,7 +380,7 @@ "space": { "required": false, "type": "string", - "description": "A way to customize which spaces (absolute, delta) are evaluted when runDelta is True. " + "description": "A way to customize which spaces (absolute, delta) are evaluated when runDelta is True." }, "maxValue": { "required": false, @@ -410,7 +410,7 @@ "encoderType": { "required": false, "type": "string", - "description": "Encoder type, for example 'ScalarEncoder, AdaptiveScalarEncoder, etc. " + "description": "Encoder type, for example 'ScalarEncoder, AdaptiveScalarEncoder, etc." } } }, diff --git a/src/nupic/swarming/permutations_runner.py b/src/nupic/swarming/permutations_runner.py index cd17ca82f9..428b0a43cb 100644 --- a/src/nupic/swarming/permutations_runner.py +++ b/src/nupic/swarming/permutations_runner.py @@ -626,10 +626,12 @@ def _launchWorkers(self, cmdLine, numWorkers): self._workers = [] for i in range(numWorkers): - stdout = tempfile.TemporaryFile() - stderr = tempfile.TemporaryFile() + stdout = tempfile.NamedTemporaryFile(delete=False) + stderr = tempfile.NamedTemporaryFile(delete=False) p = subprocess.Popen(cmdLine, bufsize=1, env=os.environ, shell=True, stdin=None, stdout=stdout, stderr=stderr) + p._stderr_file = stderr + p._stdout_file = stdout self._workers.append(p) @@ -1456,7 +1458,9 @@ def __init__(self, nupicJobID, workers): status = cjdao.ClientJobsDAO.STATUS_RUNNING else: status = cjdao.ClientJobsDAO.STATUS_COMPLETED - + if retCode != 0: + with open(worker._stderr_file.name, 'r') as err: + _emit(Verbosity.WARNING, "Job %d failed with error: %s" % (jobInfo.jobId, err.read())) jobInfo = jobInfo._replace(status=status) _emit(Verbosity.DEBUG, "JobStatus: \n%s" % pprint.pformat(jobInfo, From 5350b8b727cf0134e15a192e0270813d3a295163 Mon Sep 17 00:00:00 2001 From: Jonathan Mackenzie Date: Wed, 7 Mar 2018 12:33:07 +1030 Subject: [PATCH 10/10] Fixed metric spec schema error --- src/nupic/swarming/exp_generator/experiment_generator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/nupic/swarming/exp_generator/experiment_generator.py b/src/nupic/swarming/exp_generator/experiment_generator.py index bb394dc1a8..f0eb242540 100755 --- a/src/nupic/swarming/exp_generator/experiment_generator.py +++ b/src/nupic/swarming/exp_generator/experiment_generator.py @@ -1858,9 +1858,7 @@ def _generateMetricSpecs(options): def _generateExtraMetricSpecs(options): """Generates the non-default metrics specified by the expGenerator params """ - global _metricSpecSchema - _metricSpecSchema = {} - _metricSpecSchema['properties'] = {} + _metricSpecSchema = {'properties': {}} results = [] for metric in options['metrics']: