From eba72d0c943001e1695896242d9a5015328d1fc9 Mon Sep 17 00:00:00 2001 From: Luiz Scheinkman Date: Tue, 9 Jan 2018 23:52:02 -0800 Subject: [PATCH 1/5] Issue #1380: Add SP test checking results with no learning and 0 boosting --- src/test/unit/algorithms/SpatialPoolerTest.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index 88f3882a2e..6b8ba7eb99 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -2222,6 +2222,23 @@ namespace { EXPECT_EQ(0, countNonzero(activeColumns)); } + + TEST(SpatialPoolerTest, testSameOutputForSameInputNoLearningNoBoosting) + { + const UInt inputSize = 10; + const UInt nColumns = 20; + SpatialPooler sp; + sp.initialize({inputSize}, {nColumns}); + sp.setBoostStrength(0); + + vector input = { 1, 1, 0, 0, 1, 1, 0, 0, 1, 1 }; + vector out1(nColumns, 0); + vector out2(nColumns, 0); + sp.compute(input.data(), false, out1.data()); + sp.compute(input.data(), false, out2.data()); + EXPECT_EQ(out1, out2); + } + TEST(SpatialPoolerTest, testSaveLoad) { const char* filename = "SpatialPoolerSerialization.tmp"; From 30f9227bc7d6237c5c00b82ba2ecf62d9a7cdcf6 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 11 Jan 2018 02:40:42 +0100 Subject: [PATCH 2/5] working SP test for unstable params the unstable combination of constructor parameters can cause SP output to be different on the same input, which is clearly wrong! --- .../unit/algorithms/SpatialPoolerTest.cpp | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index 6b8ba7eb99..c873650186 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -2223,21 +2223,30 @@ namespace { } - TEST(SpatialPoolerTest, testSameOutputForSameInputNoLearningNoBoosting) + TEST(SpatialPoolerTest, testConstructorInitParamsUnstable) { - const UInt inputSize = 10; - const UInt nColumns = 20; - SpatialPooler sp; - sp.initialize({inputSize}, {nColumns}); - sp.setBoostStrength(0); - - vector input = { 1, 1, 0, 0, 1, 1, 0, 0, 1, 1 }; - vector out1(nColumns, 0); - vector out2(nColumns, 0); - sp.compute(input.data(), false, out1.data()); - sp.compute(input.data(), false, out2.data()); + /** this test exposes bug where c++ SP (wrongly) produces + different output for the same input. + XXX sensitive - marks (empirically!) discovered set of parameter + that produce the err behavior; changing any of the XXX params + may cause the SP to behave "normally" + */ + SpatialPooler sp{std::vector{10} /* input*/, std::vector{2048}/* SP output cols XXX sensitive*/, + /*pot radius*/ 20, //each col sees + /*pot pct*/ 0.5, //XXX sensitive + /*global inhibition*/ false, //XXX sensitive + /*Real localAreaDensity=*/0.02, //2% active cols + /*UInt numActiveColumnsPerInhArea=*/0, //mutex with above ^^ //XXX sensitive +}; + + vector input = { 1, 1, 0, 0, 1, 1, 0, 0, 1, 1}; + vector out1(sp.getNumColumns(), 0); + vector out2(sp.getNumColumns(), 0); + sp.compute(input.data(), true, out1.data()); + sp.compute(input.data(), true, out2.data()); EXPECT_EQ(out1, out2); - } +} + TEST(SpatialPoolerTest, testSaveLoad) { From c01a95bf128baf07f4ec1c9be4b260a91c4d6ca1 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 11 Jan 2018 14:22:38 +0100 Subject: [PATCH 3/5] SP: add check for unstable params in constructor on each initialization, SP is tested to produce stable output. --- src/nupic/algorithms/SpatialPooler.cpp | 14 ++++++++++++++ src/test/unit/algorithms/SpatialPoolerTest.cpp | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/nupic/algorithms/SpatialPooler.cpp b/src/nupic/algorithms/SpatialPooler.cpp index bd8b366173..39f29992c3 100644 --- a/src/nupic/algorithms/SpatialPooler.cpp +++ b/src/nupic/algorithms/SpatialPooler.cpp @@ -488,6 +488,17 @@ const vector& SpatialPooler::getBoostedOverlaps() const return boostedOverlaps_; } +/** helper method that checks SP output is stable for given configuration */ +bool checkUnstableParams_(SpatialPooler &sp) { + vector input(sp.getNumInputs(), 1); //auto size of input + vector out1(sp.getNumColumns(), 0); + vector out2(sp.getNumColumns(), 0); + sp.compute(input.data(), true, out1.data()); + sp.compute(input.data(), true, out2.data()); + //TODO should we add SP.reset() and call it here? + return std::equal(std::begin(out1), std::end(out1), std::begin(out2)); //compare all, element wise +} + void SpatialPooler::initialize(vector inputDimensions, vector columnDimensions, UInt potentialRadius, @@ -593,6 +604,9 @@ void SpatialPooler::initialize(vector inputDimensions, printParameters(); std::cout << "CPP SP seed = " << seed << std::endl; } + + //check for reasonable params + NTA_CHECK(checkUnstableParams_(*this)); //TODO the assert runs only at debug builds, make mandatory? } void SpatialPooler::compute(UInt inputArray[], bool learn, diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index c873650186..a380d1b866 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -2244,7 +2244,7 @@ namespace { vector out2(sp.getNumColumns(), 0); sp.compute(input.data(), true, out1.data()); sp.compute(input.data(), true, out2.data()); - EXPECT_EQ(out1, out2); + EXPECT_EQ(out1, out2); //not necessary with the check in SP initialize(), but keep here as example } From b34d1cf6fd558da03485ca91bbacfd321523ba77 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Fri, 12 Jan 2018 05:07:46 +0100 Subject: [PATCH 4/5] SP: unstable compute test: add burn in to make stable SP start --- .../unit/algorithms/SpatialPoolerTest.cpp | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index a380d1b866..d0611f2511 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -24,6 +24,8 @@ * Implementation of unit tests for SpatialPooler */ +#include //generate +#include //rand #include #include #include @@ -2223,6 +2225,9 @@ namespace { } + // function generator: + int RandomNumber01 () { return (rand()%2); } // returns random (binary) numbers from {0,1} + TEST(SpatialPoolerTest, testConstructorInitParamsUnstable) { /** this test exposes bug where c++ SP (wrongly) produces @@ -2231,20 +2236,39 @@ namespace { that produce the err behavior; changing any of the XXX params may cause the SP to behave "normally" */ - SpatialPooler sp{std::vector{10} /* input*/, std::vector{2048}/* SP output cols XXX sensitive*/, + SpatialPooler sp{std::vector{1000} /* input*/, std::vector{20}/* SP output cols XXX sensitive*/, /*pot radius*/ 20, //each col sees /*pot pct*/ 0.5, //XXX sensitive /*global inhibition*/ false, //XXX sensitive /*Real localAreaDensity=*/0.02, //2% active cols /*UInt numActiveColumnsPerInhArea=*/0, //mutex with above ^^ //XXX sensitive }; + sp.setBoostStrength(0.0); + + vector input(sp.getNumInputs(), 1); + vector out1(sp.getNumColumns(), 0); + vector out2(sp.getNumColumns(), 0); + + for(UInt i=0; i< 1800; i++) { //epochs + //burn in + for (UInt burnIn=1; burnIn < 90; burnIn++) { + generate(input.begin(), input.end(), RandomNumber01); //random input + sp.compute(input.data(), true, out1.data()); //burn in + cout<<"."; + cout.flush(); + } - vector input = { 1, 1, 0, 0, 1, 1, 0, 0, 1, 1}; - vector out1(sp.getNumColumns(), 0); - vector out2(sp.getNumColumns(), 0); - sp.compute(input.data(), true, out1.data()); - sp.compute(input.data(), true, out2.data()); - EXPECT_EQ(out1, out2); //not necessary with the check in SP initialize(), but keep here as example + //compare + for (UInt noLearn=1; noLearn < 90; noLearn++) { + generate(input.begin(), input.end(), RandomNumber01); //random input + sp.compute(input.data(), false, out1.data()); + sp.compute(input.data(), false, out2.data()); + EXPECT_EQ(out1, out2); //not necessary with the check in SP initialize(), but keep here as example + cout<<"="; + cout.flush(); + } + cout< Date: Fri, 12 Jan 2018 14:37:51 +0100 Subject: [PATCH 5/5] SP unstable output check: use no learning as it's required --- src/nupic/algorithms/SpatialPooler.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/nupic/algorithms/SpatialPooler.cpp b/src/nupic/algorithms/SpatialPooler.cpp index 39f29992c3..dbb853b877 100644 --- a/src/nupic/algorithms/SpatialPooler.cpp +++ b/src/nupic/algorithms/SpatialPooler.cpp @@ -493,9 +493,8 @@ bool checkUnstableParams_(SpatialPooler &sp) { vector input(sp.getNumInputs(), 1); //auto size of input vector out1(sp.getNumColumns(), 0); vector out2(sp.getNumColumns(), 0); - sp.compute(input.data(), true, out1.data()); - sp.compute(input.data(), true, out2.data()); - //TODO should we add SP.reset() and call it here? + sp.compute(input.data(), false, out1.data()); + sp.compute(input.data(), false, out2.data()); return std::equal(std::begin(out1), std::end(out1), std::begin(out2)); //compare all, element wise }