feat: send notifications when LND stream errors

lib/BaseClient.ts

@@ -4,7 +4,7 @@ import { ClientStatus } from './consts/Enums';
 class BaseClient extends EventEmitter {
   protected status = ClientStatus.Disconnected;
 
-  protected readonly RECONNECT_INTERVAL = 1000;
+  protected readonly RECONNECT_INTERVAL = 5000;
   protected reconnectionTimer?: any;
 
   constructor() {

lib/lightning/LndClient.ts

@@ -20,24 +20,6 @@ type LndConfig = {
   macaroonpath: string;
 };
 
-/**
- * General information about the state of this LND client
- */
-type Info = {
-  version: string;
-  syncedtochain: boolean;
-  chainsList: string[];
-  channels: ChannelCount;
-  blockheight: number;
-  uris?: string[];
-};
-
-type ChannelCount = {
-  active: number;
-  pending: number;
-  inactive?: number;
-};
-
 type SendResponse = {
   paymentPreimage: Buffer;
   paymentHash: Uint8Array | string;
@@ -65,6 +47,12 @@ interface LndClient {
 
   on(event: 'channel.backup', listener: (channelBackup: string) => void): this;
   emit(event: 'channel.backup', channelBackup: string): boolean;
+
+  on(event: 'subscription.error', listener: () => void): this;
+  emit(event: 'subscription.error'): this;
+
+  on(event: 'subscription.reconnected', listener: () => void): this;
+  emit(event: 'subscription.reconnected'): this;
 }
 
 /**
@@ -137,12 +125,12 @@ class LndClient extends BaseClient implements LndClient {
 
         this.clearReconnectTimer();
         this.setClientStatus(ClientStatus.Connected);
-
-        return true;
       } catch (error) {
         this.setClientStatus(ClientStatus.Disconnected);
-        this.logger.error(`Could not connect to ${this.symbol} ${LndClient.serviceName} at ${this.uri}` +
-        ` because: "${error.details}", retrying in ${this.RECONNECT_INTERVAL} ms`);
+
+        this.logger.error(`Could not connect to ${LndClient.serviceName} ${this.symbol} at ${this.uri}: ${formatError(error)}`);
+        this.logger.info(`Retrying in ${this.RECONNECT_INTERVAL} ms`);
+
         this.reconnectionTimer = setTimeout(this.connect, this.RECONNECT_INTERVAL);
 
         return false;
@@ -153,22 +141,27 @@ class LndClient extends BaseClient implements LndClient {
   }
 
   private reconnect = async () => {
+    this.setClientStatus(ClientStatus.Disconnected);
+
     try {
       await this.getInfo();
 
       this.logger.info(`Reestablished connection to ${LndClient.serviceName} ${this.symbol}`);
 
-      this.setClientStatus(ClientStatus.Connected);
       this.clearReconnectTimer();
 
       this.subscribePeerEvents();
       this.subscribeChannelEvents();
       this.subscribeChannelBackups();
+
+      this.setClientStatus(ClientStatus.Connected);
+      this.emit('subscription.reconnected');
     } catch (err) {
+      this.setClientStatus(ClientStatus.Disconnected);
+
       this.logger.error(`Could not reconnect to ${LndClient.serviceName} ${this.symbol}: ${err}`);
       this.logger.info(`Retrying in ${this.RECONNECT_INTERVAL} ms`);
 
-      this.setClientStatus(ClientStatus.Disconnected);
       this.reconnectionTimer = setTimeout(this.reconnect, this.RECONNECT_INTERVAL);
     }
   }
@@ -522,11 +515,20 @@ class LndClient extends BaseClient implements LndClient {
       })
       .on('end', () => deleteSubscription())
       .on('error', (error) => {
-        this.logger.error(`Invoice subscription errored: ${error.message}`);
+        this.logger.error(`${LndClient.serviceName} ${this.symbol} invoice subscription errored: ${error.message}`);
         deleteSubscription();
       });
   }
 
+  private handleSubscriptionError = async (subscriptionName: string, error: any) => {
+    this.logger.error(`${LndClient.serviceName} ${this.symbol} ${subscriptionName} subscription errored: ${formatError(error)}`);
+
+    if (this.status === ClientStatus.Connected) {
+      this.emit('subscription.error');
+      await this.reconnect();
+    }
+  }
+
   private subscribePeerEvents = () => {
     if (this.peerEventSubscription) {
       this.peerEventSubscription.cancel();
@@ -539,8 +541,7 @@ class LndClient extends BaseClient implements LndClient {
         }
       })
       .on('error', async (error) => {
-        this.logger.error(`Peer event subscription errored: ${formatError(error)}`);
-        await this.reconnect();
+        await this.handleSubscriptionError('peer event', error);
       });
   }
 
@@ -556,8 +557,7 @@ class LndClient extends BaseClient implements LndClient {
         }
       })
       .on('error', async(error) => {
-        this.logger.error(`Channel event subscription errored: ${formatError(error)}`);
-        await this.reconnect();
+        await this.handleSubscriptionError('channel event', error);
       });
   }
 
@@ -576,11 +576,10 @@ class LndClient extends BaseClient implements LndClient {
         }
       })
       .on('error', async (error) => {
-        this.logger.error(`Channel backup subscription errored: ${formatError(error)}`);
-        await this.reconnect();
+        await this.handleSubscriptionError('channel backup', error);
       });
   }
 }
 
 export default LndClient;
-export { LndConfig, SendResponse, Info };
+export { LndConfig, SendResponse };

lib/notifications/NotificationProvider.ts

@@ -72,6 +72,13 @@ class NotificationProvider {
       await this.discord.sendMessage('Started Boltz instance');
       this.logger.verbose('Connected to Discord');
 
+      for (const [, currency] of this.service.currencies) {
+        if (currency.lndClient) {
+          currency.lndClient.on('subscription.error', async () => await this.sendLostConnection(`LND ${currency.symbol}`));
+          currency.lndClient.on('subscription.reconnected', async () => await this.sendReconnected(`LND ${currency.symbol}`));
+        }
+      }
+
       const check = async () => {
         await Promise.all([
           this.checkBalances(),
@@ -102,7 +109,7 @@ class NotificationProvider {
     const promises: Promise<any>[] = [];
 
     info.getChainsMap().forEach((currency: CurrencyInfo, symbol: string) => {
-      promises.push(this.checkConnection(`${symbol} LND`, currency.getLnd()));
+      promises.push(this.checkConnection(`LND ${symbol}`, currency.getLnd()));
       promises.push(this.checkConnection(`${symbol} node`, currency.getChain()));
     });
 
@@ -112,19 +119,13 @@ class NotificationProvider {
   private checkConnection = async (service: string, object: ChainInfo | LndInfo | undefined) => {
     if (object !== undefined) {
       if (object.getError() === '') {
-        if (this.disconnected.has(service)) {
-          this.disconnected.delete(service);
-          await this.sendReconnected(service);
-        }
+        await this.sendReconnected(service);
 
         return;
       }
     }
 
-    if (!this.disconnected.has(service)) {
-      this.disconnected.add(service);
-      await this.sendLostConnection(service);
-    }
+    await this.sendLostConnection(service);
   }
 
   private checkBalances = async () => {
@@ -301,11 +302,17 @@ class NotificationProvider {
   }
 
   private sendLostConnection = async (service: string) => {
-    await this.discord.sendMessage(`**Lost connection to ${service}**`);
+    if (!this.disconnected.has(service)) {
+      this.disconnected.add(service);
+      await this.discord.sendMessage(`**Lost connection to ${service}**`);
+    }
   }
 
   private sendReconnected = async (service: string) => {
-    await this.discord.sendMessage(`Reconnected to ${service}`);
+    if (this.disconnected.has(service)) {
+      this.disconnected.delete(service);
+      await this.discord.sendMessage(`Reconnected to ${service}`);
+    }
   }
 
   private formatBalances = (balance: number, threshold: number) => {

lib/service/Service.ts

@@ -71,7 +71,7 @@ class Service {
     private logger: Logger,
     config: ConfigType,
     private walletManager: WalletManager,
-    private currencies: Map<string, Currency>,
+    public currencies: Map<string, Currency>,
   ) {
     this.prepayMinerFee = config.prepayminerfee;
     this.logger.debug(`Prepay miner fee for Reverse Swaps is ${this.prepayMinerFee ? 'enabled' : 'disabled' }`);

package.json

@@ -94,8 +94,8 @@
     "@types/web3": "^1.2.2",
     "@types/yargs": "^15.0.5",
     "@types/zeromq": "^4.6.3",
-    "@typescript-eslint/eslint-plugin": "^3.9.0",
-    "@typescript-eslint/parser": "^3.9.0",
+    "@typescript-eslint/eslint-plugin": "^3.9.1",
+    "@typescript-eslint/parser": "^3.9.1",
     "concurrently": "^5.3.0",
     "conventional-changelog": "^3.1.23",
     "conventional-changelog-cli": "^2.1.0",
@@ -107,7 +107,7 @@
     "grpc-tools": "^1.9.1",
     "grpc_tools_node_protoc_ts": "^4.1.3",
     "jest": "26.4.0",
-    "truffle": "^5.1.40",
+    "truffle": "^5.1.41",
     "ts-jest": "26.2.0",
     "ts-node": "8.10.2",
     "ts-protoc-gen": "^0.12.0",

test/unit/notifications/NotificationProvider.spec.ts

@@ -41,6 +41,7 @@ jest.mock('../../../lib/service/Service', () => {
           }
         },
       },
+      currencies: new Map<string, any>(),
       getInfo: mockGetInfo,
       getBalance: mockGetBalance,
     };