diff --git a/app/persistence/fabric/CRUDService.ts b/app/persistence/fabric/CRUDService.ts index fa78e7597..cf23d74e1 100644 --- a/app/persistence/fabric/CRUDService.ts +++ b/app/persistence/fabric/CRUDService.ts @@ -499,6 +499,115 @@ export class CRUDService { return row; } + /** + * + * Delete the old data based on time purge. + * + * @param {*} channel_genesis_hash + * @param {*} daysToPurge + * @returns + * @memberof CRUDService + */ + async deleteOldData(channel_genesis_hash: string, network_id: string, daysToPurge: number) { + //Fetching the blockto and block from value as per the days required + let blockfrom = await this.sql.updateBySql( + `SELECT CASE WHEN MIN(blocknum) is null THEN 0 ELSE MIN(blocknum) end as blockfrom FROM blocks WHERE createdt < CURRENT_DATE - INTERVAL '1 day' *$1 and channel_genesis_hash=$2 and network_name = $3`, + [daysToPurge, channel_genesis_hash, network_id]); + let blockto = await this.sql.updateBySql( + `SELECT CASE WHEN MAX(blocknum) is null THEN 0 ELSE MAX(blocknum) end as blockto FROM blocks WHERE createdt < CURRENT_DATE - INTERVAL '1 day' *$1 and channel_genesis_hash=$2 and network_name = $3`, + [daysToPurge, channel_genesis_hash, network_id]); + if (blockto[0].blockto != 0) { + //Deleting the txn and blocks table + await this.sql.updateBySql( + `DELETE FROM transactions WHERE blockid>= $1 and blockid<=$2 and channel_genesis_hash=$3 and network_name = $4`, + [blockfrom[0].blockfrom, blockto[0].blockto, channel_genesis_hash, network_id] + ); + await this.sql.updateBySql( + `DELETE FROM blocks WHERE blocknum>= $1 and blocknum<=$2 and channel_genesis_hash=$3 and network_name = $4`, + [blockfrom[0].blockfrom, blockto[0].blockto, channel_genesis_hash, network_id] + ); + await this.explorerTableUpdation( + blockfrom[0].blockfrom, blockto[0].blockto, "TIME", channel_genesis_hash, network_id); + return true; + } + return false; + } + + /** + * + * Fetch block to value from explorer_audit table. + * + * @param {*} channel_genesis_hash + * @param {*} mode + * @returns + * @memberof CRUDService + */ + async fetchLastBlockToFromExplorerAudit(mode: string, channel_genesis_hash: string, network_id: string) { + let blocktoValue = await this.sql.updateBySql( + `SELECT blockto FROM explorer_audit where mode =$1 and channel_genesis_hash=$2 and network_name = $3`, [mode, channel_genesis_hash, network_id]); + return blocktoValue[0].blockto; + + } + + /** + * + * Delete the old data based on block count purge. + * + * @param {*} channel_genesis_hash + * @param {*} blockCount + * @returns + * @memberof CRUDService + */ + async deleteBlock(network_name: string, channel_genesis_hash: string, blockCount: number) { + const count: any = await this.sql.getRowsBySQlCase( + ' select count(*) as count from blocks where channel_genesis_hash=$1 and network_name = $2 ', + [channel_genesis_hash, network_name] + ); + const rowCount: number = count.count; + let rowsToDelete: number = 0; + if (rowCount > blockCount) { + rowsToDelete = rowCount - blockCount; + } + if (rowsToDelete > 0) { + let blockfrom = await this.sql.updateBySql( + `SELECT min(blocknum) as blockfrom FROM blocks WHERE channel_genesis_hash=$1 and network_name = $2`, [channel_genesis_hash, network_name]); + let blockto = await this.sql.updateBySql( + `SELECT max(blocknum) as blockto FROM blocks WHERE channel_genesis_hash=$1 and network_name = $2`, [channel_genesis_hash, network_name]); + await this.sql.updateBySql( + `DELETE FROM transactions WHERE blockid>= $1 and blockid<=$2 and channel_genesis_hash=$3 and network_name = $4`, + [blockfrom[0].blockfrom, blockto[0].blockto - blockCount, channel_genesis_hash, network_name] + ); + await this.sql.updateBySql( + `DELETE FROM blocks WHERE blocknum>= $1 and blocknum<=$2 and channel_genesis_hash=$3 and network_name = $4`, + [blockfrom[0].blockfrom, blockto[0].blockto - blockCount, channel_genesis_hash, network_name] + ); + await this.explorerTableUpdation( + blockfrom[0].blockfrom, blockto[0].blockto - blockCount, "BLOCKCOUNT", channel_genesis_hash, network_name); + + return true; + } + return false; + } + + /** + * + * Update the explorer_audit table based on the details of purge. + * + * @param {*} channel_genesis_hash + * @param {*} blockFrom + * @param {*} blockTo + * @param {*} purgeMode + * @returns + * @memberof CRUDService + */ + async explorerTableUpdation(blockFrom: number, blockTo: number, purgeMode: string, channel_genesis_hash: string, network_name: string) { + const updateReponse = await this.sql.updateBySql( + `insert into explorer_audit (lastupdated,status,blockfrom,blockto,mode,channel_genesis_hash,network_name) values ($1,$2,$3,$4,$5,$6,$7) on conflict(mode,channel_genesis_hash,network_name) + do update set lastupdated = $1, status = $2, blockfrom = $3, blockto = $4, mode = $5, channel_genesis_hash =$6,network_name= $7 `, + [new Date(), "SUCCESS", blockFrom, blockTo, purgeMode, channel_genesis_hash, network_name]); + logger.info(" Data added to explorer_audit table", updateReponse); + } + } /** diff --git a/app/persistence/fabric/postgreSQL/db/explorerpg.sql b/app/persistence/fabric/postgreSQL/db/explorerpg.sql index 5af004306..4343abbbb 100644 --- a/app/persistence/fabric/postgreSQL/db/explorerpg.sql +++ b/app/persistence/fabric/postgreSQL/db/explorerpg.sql @@ -205,6 +205,26 @@ CREATE TABLE users ); ALTER table users owner to :user; +-- --------------------------- +-- Table structure for `explorer audit` +-- ---------------------------- +DROP TABLE IF EXISTS explorer_audit; +CREATE TYPE purgeMode AS ENUM('TIME','BLOCKCOUNT'); +CREATE TABLE explorer_audit +( + id SERIAL PRIMARY KEY, + lastupdated Timestamp DEFAULT NULL, + status varchar(255) DEFAULT NULL, + blockfrom integer DEFAULT NULL, + blockto integer DEFAULT NULL, + mode purgeMode DEFAULT NULL, + channel_genesis_hash character varying(256) DEFAULT NULL, + network_name varchar(255) +); +CREATE UNIQUE INDEX exp_audit_id +ON explorer_audit (network_name,mode,channel_genesis_hash); +ALTER table explorer_audit owner to :user; + DROP TABLE IF EXISTS write_lock; CREATE TABLE write_lock ( diff --git a/app/platform/fabric/config.json b/app/platform/fabric/config.json index f99d37b59..6f5a28f26 100644 --- a/app/platform/fabric/config.json +++ b/app/platform/fabric/config.json @@ -2,7 +2,10 @@ "network-configs": { "test-network": { "name": "Test Network", - "profile": "./connection-profile/test-network.json" + "profile": "./connection-profile/test-network.json", + "blockCount": 5, + "purgeMode":"NONE", + "daysToPurge": 7 } }, "license": "Apache-2.0" diff --git a/app/platform/fabric/sync/SyncService.ts b/app/platform/fabric/sync/SyncService.ts index b4e7dac96..978fb1b0d 100644 --- a/app/platform/fabric/sync/SyncService.ts +++ b/app/platform/fabric/sync/SyncService.ts @@ -12,9 +12,16 @@ import { explorerError } from '../../../common/ExplorerMessage'; import * as FabricConst from '../../../platform/fabric/utils/FabricConst'; import * as FabricUtils from '../../../platform/fabric/utils/FabricUtils'; +import * as path from 'path'; +import * as fs from 'fs'; + const logger = helper.getLogger('SyncServices'); const fabric_const = FabricConst.fabric.const; +const purge_mode = FabricConst.PurgeModes; +const config_path = path.resolve(__dirname, '../config.json'); +const all_config = JSON.parse(fs.readFileSync(config_path, 'utf8')); +const network_configs = all_config[fabric_const.NETWORK_CONFIGS]; // Transaction validation code const _validation_codes = {}; @@ -361,6 +368,10 @@ export class SyncServices { // Get channel information from ledger const channelInfo = await client.fabricGateway.queryChainInfo(channel_name); + //Get purge configuration from config.json + const purgeMode = network_configs[network_id].purgeMode.toUpperCase(); + const daysToPurge = network_configs[network_id].daysToPurge; + const blockCount = network_configs[network_id].blockCount; if (!channelInfo) { logger.info(`syncBlocks: Failed to retrieve channelInfo >> ${channel_name}`); @@ -390,7 +401,20 @@ export class SyncServices { result.missing_id ); if (block) { - await this.processBlockEvent(client, block, noDiscovery); + if (this.validatePurgemode(purgeMode)) { + const lastBlockTo = await this.persistence.getCrudService() + .fetchLastBlockToFromExplorerAudit(purgeMode, channel_genesis_hash, network_id); + if (result.missing_id > lastBlockTo) { + await this.processBlockEvent(client, block, noDiscovery); + } + else { + logger.info(`Not saving this block to DB since we are keeping only recent data # ${result.missing_id}`); + } + } + else { + await this.processBlockEvent(client, block, noDiscovery); + logger.debug("Purge unsuccessful since purge inputs are not matching"); + } } } catch { logger.error(`Failed to process Block # ${result.missing_id}`); @@ -399,6 +423,24 @@ export class SyncServices { } else { logger.debug('Missing blocks not found for %s', channel_name); } + if (this.validatePurgemode(purgeMode)) { + let response: boolean; + switch (purgeMode) { + case purge_mode[0]: { + response = await this.persistence. + getCrudService(). + deleteOldData(channel_genesis_hash, network_id, daysToPurge); + break; + } + case purge_mode[1]: { + response = await this.persistence + .getCrudService() + .deleteBlock(network_id, channel_genesis_hash, blockCount); + break; + } + } + logger.info(`Result of Purging based on ${purgeMode} :`, response); + } const index = this.synchInProcess.indexOf(synch_key); this.synchInProcess.splice(index, 1); logger.info(`syncBlocks: Finish >> ${synch_key}`); @@ -736,6 +778,13 @@ export class SyncServices { } } + validatePurgemode(purgeMode: string) { + if (Object.values(purge_mode).includes(purgeMode)) { + return true; + } + else return false; + } + /** * * diff --git a/app/platform/fabric/utils/FabricConst.ts b/app/platform/fabric/utils/FabricConst.ts index 72517bfe7..8ac13c3a6 100644 --- a/app/platform/fabric/utils/FabricConst.ts +++ b/app/platform/fabric/utils/FabricConst.ts @@ -18,3 +18,8 @@ export const fabric = { NOTITY_TYPE_CLIENTERROR: '6' } }; + +export enum PurgeModes { + 'TIME', + 'BLOCKCOUNT' +}; \ No newline at end of file