Migrating data to datastore

CHANGELOG.md

@@ -11,6 +11,10 @@ All notable changes to this project will be documented in this file.
 * Fixed `addModule` function to be backwards compatible and call the new `addModuleWithLabel` function with an empty label.
 * Fixed event `ModuleAdded` to also emit `_label`.    
 * Fixed function `getModule` to also return the respective module label.
+* Added datastore that is used to store data like investor list that is shared among modules.
+* `getInvestorCount()` now returns length of investor array that is everyone who ever held some st or has kyc data attached.
+* `holderCount()` returns the number of current st holders.
+* Added flags for Investors. Accredited and canbuyfromsto are now flags
 
 ## STR
 * Introduce new contract `STRGetter.sol`. It only contains the getter functions of the STR.
@@ -19,7 +23,9 @@ All notable changes to this project will be documented in this file.
 * Removed `_polyToken` parameter from `initialize` function in `SecurityTokenRegistry`.
 
 ## GeneralTransferManager
-* Add `_isAccredited` variable in the `modifyWhitelist()` function of the GeneralTransferManager.
+* `modifyWhitelist()` function renamed to `modifyKYCData()`.
+* Added functions to modify and get flags
+* `canBuyFromSto` is now `canNotBuyFromSto` and it is the flag `1`
 
 ## Generalize
 * Removed `_polyAddress` parameter from constructors of all modules and module factories.

contracts/datastore/DataStoreStorage.sol

@@ -16,6 +16,6 @@ contract DataStoreStorage {
     mapping (bytes32 => address[]) internal addressArrayData;
     mapping (bytes32 => bool[]) internal boolArrayData;
 
-    uint8 constant DATA_KEY = 6;
-    bytes32 public constant MANAGEDATA = "MANAGEDATA";
+    uint8 internal constant DATA_KEY = 6;
+    bytes32 internal constant MANAGEDATA = "MANAGEDATA";
 }

contracts/interfaces/IDataStore.sol

@@ -125,4 +125,12 @@ interface IDataStore {
     function getAddressArrayElement(bytes32 _key, uint256 _index) external view returns(address);
 
     function getBoolArrayElement(bytes32 _key, uint256 _index) external view returns(bool);
+
+    function getUint256ArrayElements(bytes32 _key, uint256 _startIndex, uint256 _endIndex) external view returns(uint256[] memory);
+
+    function getBytes32ArrayElements(bytes32 _key, uint256 _startIndex, uint256 _endIndex) external view returns(bytes32[] memory);
+
+    function getAddressArrayElements(bytes32 _key, uint256 _startIndex, uint256 _endIndex) external view returns(address[] memory);
+
+    function getBoolArrayElements(bytes32 _key, uint256 _startIndex, uint256 _endIndex) external view returns(bool[] memory);
 }

contracts/interfaces/ISecurityToken.sol

@@ -316,6 +316,11 @@ interface ISecurityToken {
      */
     function getInvestorCount() external view returns(uint256);
 
+    /**
+     * @notice Gets the holder count (investors with non zero balance)
+     */
+    function holderCount() external view returns(uint256);
+
     /**
       * @notice Overloaded version of the transfer function
       * @param _to receiver of transfer

contracts/libraries/TokenLib.sol

@@ -2,10 +2,14 @@ pragma solidity ^0.5.0;
 
 import "openzeppelin-solidity/contracts/math/SafeMath.sol";
 import "../interfaces/IPoly.sol";
+import "../interfaces/IDataStore.sol";
 
 library TokenLib {
     using SafeMath for uint256;
 
+    bytes32 internal constant WHITELIST = "WHITELIST";
+    bytes32 internal constant INVESTORSKEY = 0xdf3a8dd24acdd05addfc6aeffef7574d2de3f844535ec91e8e0f3e45dba96731; //keccak256(abi.encodePacked("INVESTORS"))
+
     // Struct for module data
     struct ModuleData {
         bytes32 name;
@@ -24,15 +28,6 @@ library TokenLib {
         uint256 value;
     }
 
-    struct InvestorDataStorage {
-        // List of investors who have ever held a non-zero token balance
-        mapping(address => bool) investorListed;
-        // List of token holders
-        address[] investors;
-        // Total number of non-zero token holders
-        uint256 investorCount;
-    }
-
     // Emit when Module is archived from the SecurityToken
     event ModuleArchived(uint8[] _types, address _module);
     // Emit when Module is unarchived from the SecurityToken
@@ -217,40 +212,54 @@ library TokenLib {
 
     /**
     * @notice Keeps track of the number of non-zero token holders
-    * @param _investorData Date releated to investor metrics
+    * @param _holderCount Number of current token holders
     * @param _from Sender of transfer
     * @param _to Receiver of transfer
     * @param _value Value of transfer
     * @param _balanceTo Balance of the _to address
     * @param _balanceFrom Balance of the _from address
+    * @param _dataStore address of data store
     */
     function adjustInvestorCount(
-        InvestorDataStorage storage _investorData,
+        uint256 _holderCount,
         address _from,
         address _to,
         uint256 _value,
         uint256 _balanceTo,
-        uint256 _balanceFrom
+        uint256 _balanceFrom,
+        address _dataStore
     )
         public
+        returns(uint256)
     {
         if ((_value == 0) || (_from == _to)) {
-            return;
+            return _holderCount;
         }
         // Check whether receiver is a new token holder
         if ((_balanceTo == 0) && (_to != address(0))) {
-            _investorData.investorCount = (_investorData.investorCount).add(1);
+            _holderCount = _holderCount.add(1);
+            IDataStore dataStore = IDataStore(_dataStore);
+            if (!_isExistingInvestor(_to, dataStore)) {
+                dataStore.insertAddress(INVESTORSKEY, _to);
+                //KYC data can not be present if added is false and hence we can set packed KYC as uint256(1) to set added as true
+                dataStore.setUint256(_getKey(WHITELIST, _to), uint256(1));
+            }
         }
         // Check whether sender is moving all of their tokens
         if (_value == _balanceFrom) {
-            _investorData.investorCount = (_investorData.investorCount).sub(1);
-        }
-        //Also adjust investor list
-        if (!_investorData.investorListed[_to] && (_to != address(0))) {
-            _investorData.investors.push(_to);
-            _investorData.investorListed[_to] = true;
+            _holderCount = _holderCount.sub(1);
         }
 
+        return _holderCount;
+    }
+
+    function _getKey(bytes32 _key1, address _key2) internal pure returns(bytes32) {
+        return bytes32(keccak256(abi.encodePacked(_key1, _key2)));
     }
 
+    function _isExistingInvestor(address _investor, IDataStore dataStore) internal view returns(bool) {
+        uint256 data = dataStore.getUint256(_getKey(WHITELIST, _investor));
+        //extracts `added` from packed `whitelistData`
+        return uint8(data) == 0 ? false : true;
+    }
 }

contracts/libraries/VersionUtils.sol

@@ -110,21 +110,22 @@ library VersionUtils {
     /**
      * @notice Used to packed the KYC data
      */
-    function packKYC(uint64 _a, uint64 _b, uint64 _c, uint8 _d, uint8 _e, uint8 _f) internal pure returns(uint256) {
-        return (uint256(_a) << 152) | (uint256(_b) << 88) | (uint256(_c) << 24) | (uint256(_d) << 16) | (uint256(_e) << 8) | uint256(_f);
+    function packKYC(uint64 _a, uint64 _b, uint64 _c, uint8 _d) internal pure returns(uint256) {
+        // this function packs 3 uint64 and a uint8 together in a uint256 to save storage cost
+        // a is rotated left by 136 bits, b is rotated left by 72 bits and c is rotated left by 8 bits.
+        // rotation pads empty bits with zeroes so now we can safely do a bitwise OR operation to pack
+        // all the variables together.
+        return (uint256(_a) << 136) | (uint256(_b) << 72) | (uint256(_c) << 8) | uint256(_d);
     }
 
     /**
      * @notice Used to convert packed data into KYC data
      * @param _packedVersion Packed data
      */
-    function unpackKYC(uint256 _packedVersion) internal pure returns(uint64 fromTime, uint64 toTime, uint64 expiryTime, uint8 canBuy, uint8 added, uint8 accredited) {
-        fromTime = uint64(_packedVersion >> 152);
-        toTime = uint64(_packedVersion >> 88);
-        expiryTime = uint64(_packedVersion >> 24);
-        canBuy = uint8(_packedVersion >> 16);
-        added = uint8(_packedVersion >> 8);
-        accredited = uint8(_packedVersion);
+    function unpackKYC(uint256 _packedVersion) internal pure returns(uint64 fromTime, uint64 toTime, uint64 expiryTime, uint8 added) {
+        fromTime = uint64(_packedVersion >> 136);
+        toTime = uint64(_packedVersion >> 72);
+        expiryTime = uint64(_packedVersion >> 8);
+        added = uint8(_packedVersion);
     }
-
 }

contracts/modules/STO/CappedSTO.sol

@@ -208,10 +208,10 @@ contract CappedSTO is CappedSTOStorage, STO, ReentrancyGuard {
       Observe state and use revert statements to undo rollback when valid conditions are not met.
     */
     function _postValidatePurchase(
-        address, /*_beneficiary*/
+        address _beneficiary,
         uint256 /*_investedAmount*/
-    ) internal pure {
-        // optional override
+    ) internal view {
+        require(_canBuy(_beneficiary), "Unauthorized");
     }
 
     /**

contracts/modules/STO/DummySTO.sol

@@ -47,6 +47,7 @@ contract DummySTO is DummySTOStorage, STO {
     function generateTokens(address _investor, uint256 _amount) public withPerm(ADMIN) {
         require(!paused, "Should not be paused");
         require(_amount > 0, "Amount should be greater than 0");
+        require(_canBuy(_investor), "Unauthorized");
         ISecurityToken(securityToken).mint(_investor, _amount);
         if (investors[_investor] == 0) {
             investorCount = investorCount + 1;

contracts/modules/STO/PreSaleSTO.sol

@@ -71,13 +71,14 @@ contract PreSaleSTO is PreSaleSTOStorage, STO {
         uint256 _amount,
         uint256 _etherContributed,
         uint256 _polyContributed
-    ) 
-        public 
-        withPerm(PRE_SALE_ADMIN) 
+    )
+        public
+        withPerm(PRE_SALE_ADMIN)
     {
         /*solium-disable-next-line security/no-block-members*/
         require(now <= endTime, "Already passed Endtime");
         require(_amount > 0, "No. of tokens provided should be greater the zero");
+        require(_canBuy(_investor), "Unauthorized");
         ISecurityToken(securityToken).mint(_investor, _amount);
         if (investors[_investor] == uint256(0)) {
             investorCount = investorCount.add(1);
@@ -101,9 +102,9 @@ contract PreSaleSTO is PreSaleSTOStorage, STO {
         uint256[] memory _amounts,
         uint256[] memory _etherContributed,
         uint256[] memory _polyContributed
-    ) 
-        public 
-        withPerm(PRE_SALE_ADMIN) 
+    )
+        public
+        withPerm(PRE_SALE_ADMIN)
     {
         require(_investors.length == _amounts.length, "Mis-match in length of the arrays");
         require(_etherContributed.length == _polyContributed.length, "Mis-match in length of the arrays");

contracts/modules/STO/STO.sol

@@ -70,4 +70,13 @@ contract STO is ISTO, STOStorage, Module, Pausable {
         emit SetFundRaiseTypes(_fundRaiseTypes);
     }
 
+    function _canBuy(address _investor) internal view returns(bool) {
+        IDataStore dataStore = IDataStore(getDataStore());
+        uint256 flags = dataStore.getUint256(_getKey(INVESTORFLAGS, _investor));
+        return(flags & (uint256(1) << 1) == 0);
+    }
+
+    function _getKey(bytes32 _key1, address _key2) internal pure returns(bytes32) {
+        return bytes32(keccak256(abi.encodePacked(_key1, _key2)));
+    }
 }

contracts/modules/STO/STOStorage.sol

@@ -5,6 +5,7 @@ pragma solidity ^0.5.0;
  */
 
 contract STOStorage {
+    bytes32 internal constant INVESTORFLAGS = "INVESTORFLAGS";
 
     mapping (uint8 => bool) public fundRaiseTypes;
     mapping (uint8 => uint256) public fundsRaised;

contracts/modules/STO/USDTieredSTO.sol

@@ -199,7 +199,7 @@ contract USDTieredSTO is USDTieredSTOStorage, STO {
         internal
     {
         require(
-        	_tokensPerTierTotal.length > 0 &&
+            _tokensPerTierTotal.length > 0 &&
             _ratePerTier.length == _tokensPerTierTotal.length &&
             _ratePerTierDiscountPoly.length == _tokensPerTierTotal.length &&
             _tokensPerTierDiscountPoly.length == _tokensPerTierTotal.length,
@@ -272,24 +272,6 @@ contract USDTieredSTO is USDTieredSTOStorage, STO {
         totalTokensSold = tempSold;
     }
 
-    /**
-     * @notice Modifies the list of accredited addresses
-     * @param _investors Array of investor addresses to modify
-     * @param _accredited Array of bools specifying accreditation status
-     */
-    function changeAccredited(address[] memory _investors, bool[] memory _accredited) public onlyOwner {
-        require(_investors.length == _accredited.length, "Array length mismatch");
-        for (uint256 i = 0; i < _investors.length; i++) {
-            if (_accredited[i]) {
-                investors[_investors[i]].accredited = uint8(1);
-            } else {
-                investors[_investors[i]].accredited = uint8(0);
-            }
-            _addToInvestorsList(_investors[i]);
-            emit SetAccredited(_investors[i], _accredited[i]);
-        }
-    }
-
     /**
      * @notice Modifies the list of overrides for non-accredited limits in USD
      * @param _investors Array of investor addresses to modify
@@ -299,34 +281,26 @@ contract USDTieredSTO is USDTieredSTOStorage, STO {
         //nonAccreditedLimitUSDOverride
         require(_investors.length == _nonAccreditedLimit.length, "Array length mismatch");
         for (uint256 i = 0; i < _investors.length; i++) {
-            investors[_investors[i]].nonAccreditedLimitUSDOverride = _nonAccreditedLimit[i];
-            _addToInvestorsList(_investors[i]);
+            nonAccreditedLimitUSDOverride[_investors[i]] = _nonAccreditedLimit[i];
             emit SetNonAccreditedLimit(_investors[i], _nonAccreditedLimit[i]);
         }
     }
 
-    function _addToInvestorsList(address _investor) internal {
-        if (investors[_investor].seen == uint8(0)) {
-            investors[_investor].seen = uint8(1);
-            investorsList.push(_investor);
-        }
-    }
-
     /**
      * @notice Returns investor accredited & non-accredited override informatiomn
-     * @return address[] list of all configured investors
-     * @return bool[] whether investor is accredited
-     * @return uint256[] any USD overrides for non-accredited limits for the investor
+     * @return investors list of all configured investors
+     * @return accredited whether investor is accredited
+     * @return override any USD overrides for non-accredited limits for the investor
      */
-    function getAccreditedData() external view returns (address[] memory, bool[] memory, uint256[] memory) {
-        bool[] memory accrediteds = new bool[](investorsList.length);
-        uint256[] memory nonAccreditedLimitUSDOverrides = new uint256[](investorsList.length);
-        uint256 i;
-        for (i = 0; i < investorsList.length; i++) {
-            accrediteds[i] = (investors[investorsList[i]].accredited == uint8(0)? false: true);
-            nonAccreditedLimitUSDOverrides[i] = investors[investorsList[i]].nonAccreditedLimitUSDOverride;
+    function getAccreditedData() external view returns (address[] memory investors, bool[] memory accredited, uint256[] memory overrides) {
+        IDataStore dataStore = IDataStore(getDataStore());
+        investors = dataStore.getAddressArray(INVESTORSKEY);
+        accredited = new bool[](investors.length);
+        overrides = new uint256[](investors.length);
+        for (uint256 i = 0; i < investors.length; i++) {
+            accredited[i] = _getIsAccredited(investors[i], dataStore);
+            overrides[i] = nonAccreditedLimitUSDOverride[investors[i]];
         }
-        return (investorsList, accrediteds, nonAccreditedLimitUSDOverrides);
     }
 
     /**
@@ -443,6 +417,8 @@ contract USDTieredSTO is USDTieredSTOStorage, STO {
             require(_beneficiary == msg.sender, "Beneficiary != funder");
         }
 
+        require(_canBuy(_beneficiary), "Unauthorized");
+
         uint256 originalUSD = DecimalMath.mul(_rate, _investmentValue);
         uint256 allowedUSD = _buyTokensChecks(_beneficiary, _investmentValue, originalUSD);
 
@@ -489,8 +465,8 @@ contract USDTieredSTO is USDTieredSTOStorage, STO {
         require(investedUSD.add(investorInvestedUSD[_beneficiary]) >= minimumInvestmentUSD, "Investment < min");
         netInvestedUSD = investedUSD;
         // Check for non-accredited cap
-        if (investors[_beneficiary].accredited == uint8(0)) {
-            uint256 investorLimitUSD = (investors[_beneficiary].nonAccreditedLimitUSDOverride == 0) ? nonAccreditedLimitUSD : investors[_beneficiary].nonAccreditedLimitUSDOverride;
+        if (!_isAccredited(_beneficiary)) {
+            uint256 investorLimitUSD = (nonAccreditedLimitUSDOverride[_beneficiary] == 0) ? nonAccreditedLimitUSD : nonAccreditedLimitUSDOverride[_beneficiary];
             require(investorInvestedUSD[_beneficiary] < investorLimitUSD, "Over Non-accredited investor limit");
             if (investedUSD.add(investorInvestedUSD[_beneficiary]) > investorLimitUSD)
                 netInvestedUSD = investorLimitUSD.sub(investorInvestedUSD[_beneficiary]);
@@ -568,6 +544,17 @@ contract USDTieredSTO is USDTieredSTOStorage, STO {
         }
     }
 
+    function _isAccredited(address _investor) internal view returns(bool) {
+        IDataStore dataStore = IDataStore(getDataStore());
+        return _getIsAccredited(_investor, dataStore);
+    }
+
+    function _getIsAccredited(address _investor, IDataStore dataStore) internal view returns(bool) {
+        uint256 flags = dataStore.getUint256(_getKey(INVESTORFLAGS, _investor));
+        uint256 flag = flags & uint256(1); //isAccredited is flag 0 so we don't need to bit shift flags.
+        return flag > 0 ? true : false;
+    }
+
     /////////////
     // Getters //
     /////////////
@@ -577,7 +564,7 @@ contract USDTieredSTO is USDTieredSTOStorage, STO {
      * @return bool Whether the STO is accepting investments
      */
     function isOpen() public view returns(bool) {
-    	/*solium-disable-next-line security/no-block-members*/
+        /*solium-disable-next-line security/no-block-members*/
         if (isFinalized || now < startTime || now >= endTime || capReached())
             return false;
         return true;

contracts/modules/TransferManager/CountTransferManager.sol

@@ -34,7 +34,7 @@ contract CountTransferManager is CountTransferManagerStorage, TransferManager {
         returns(Result)
     {
         if (!paused) {
-            if (maxHolderCount < ISecurityToken(securityToken).getInvestorCount()) {
+            if (maxHolderCount < ISecurityToken(securityToken).holderCount()) {
                 // Allow transfers to existing maxHolders
                 if (ISecurityToken(securityToken).balanceOf(_to) != 0 || ISecurityToken(securityToken).balanceOf(_from) == _amount) {
                     return Result.NA;