fix!: allow safe leverage operations during partial oracle outages

CHANGELOG.md

@@ -55,6 +55,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 - [1812](https://github.com/umee-network/umee/pull/1812) MaxCollateralShare now works during partial oracle outages when certain conditions are safe.
 - [1736](https://github.com/umee-network/umee/pull/1736) Blacklisted tokens no longer add themselves back to the oracle accept list.
 - [1807](https://github.com/umee-network/umee/pull/1807) Fixes BNB ibc denom in 4.1 migration
+- [1821](https://github.com/umee-network/umee/pull/1821) Allow safe leverage operations during partial oracle outages.
 
 ## [v4.0.1](https://github.com/umee-network/umee/releases/tag/v4.0.1) - 2023-02-10
 

proto/umee/leverage/v1/query.proto

@@ -222,31 +222,40 @@ message QueryAccountSummary {
 // QueryAccountSummaryResponse defines the response structure for the AccountSummary gRPC service handler.
 message QueryAccountSummaryResponse {
   // Supplied Value is the sum of the USD value of all tokens the account has supplied, including interest earned.
+  // Computation skips assets which are missing oracle prices, potentially resulting in a lower supplied
+  // value than if prices were all available.
   string supplied_value = 1 [
     (gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
     (gogoproto.nullable)   = false
   ];
   // Collateral Value is the sum of the USD value of all uTokens the account has collateralized.
+  // Computation skips collateral which is missing an oracle price, potentially resulting in a lower collateral
+  // value than if prices were all available.
   string collateral_value = 2 [
     (gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
     (gogoproto.nullable)   = false
   ];
   // Borrowed Value is the sum of the USD value of all tokens the account has borrowed, including interest owed.
   // It always uses spot prices.
+  // Computation skips borrows which are missing oracle prices, potentially resulting in a lower borrowed
+  // value than if prices were all available.
   string borrowed_value = 3 [
     (gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
     (gogoproto.nullable)   = false
   ];
   // Borrow Limit is the maximum Borrowed Value the account is allowed to reach through direct borrowing.
   // The lower of spot or historic price for each collateral token is used when calculating borrow limits.
+  // Computation skips collateral which is missing an oracle price, potentially resulting in a lower borrow
+  // limit than if prices were all available.
   string borrow_limit = 4 [
     (gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
     (gogoproto.nullable)   = false
   ];
   // Liquidation Threshold is the Borrowed Value at which the account becomes eligible for liquidation.
+  // Will be null if an oracle price required for computation is missing.
   string liquidation_threshold = 5 [
     (gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
-    (gogoproto.nullable)   = false
+    (gogoproto.nullable)   = true
   ];
 }
 

swagger/swagger.yaml

@@ -132,18 +132,33 @@ paths:
                 description: >-
                   Supplied Value is the sum of the USD value of all tokens the
                   account has supplied, including interest earned.
+
+                  Computation skips assets which are missing oracle prices,
+                  potentially resulting in a lower supplied
+
+                  value than if prices were all available.
               collateral_value:
                 type: string
                 description: >-
                   Collateral Value is the sum of the USD value of all uTokens
                   the account has collateralized.
+
+                  Computation skips collateral which is missing an oracle price,
+                  potentially resulting in a lower collateral
+
+                  value than if prices were all available.
               borrowed_value:
                 type: string
                 description: >-
                   Borrowed Value is the sum of the USD value of all tokens the
                   account has borrowed, including interest owed.
 
                   It always uses spot prices.
+
+                  Computation skips borrows which are missing oracle prices,
+                  potentially resulting in a lower borrowed
+
+                  value than if prices were all available.
               borrow_limit:
                 type: string
                 description: >-
@@ -152,11 +167,19 @@ paths:
 
                   The lower of spot or historic price for each collateral token
                   is used when calculating borrow limits.
+
+                  Computation skips collateral which is missing an oracle price,
+                  potentially resulting in a lower borrow
+
+                  limit than if prices were all available.
               liquidation_threshold:
                 type: string
                 description: >-
                   Liquidation Threshold is the Borrowed Value at which the
                   account becomes eligible for liquidation.
+
+                  Will be null if an oracle price required for computation is
+                  missing.
             description: >-
               QueryAccountSummaryResponse defines the response structure for the
               AccountSummary gRPC service handler.
@@ -2023,18 +2046,33 @@ definitions:
         description: >-
           Supplied Value is the sum of the USD value of all tokens the account
           has supplied, including interest earned.
+
+          Computation skips assets which are missing oracle prices, potentially
+          resulting in a lower supplied
+
+          value than if prices were all available.
       collateral_value:
         type: string
         description: >-
           Collateral Value is the sum of the USD value of all uTokens the
           account has collateralized.
+
+          Computation skips collateral which is missing an oracle price,
+          potentially resulting in a lower collateral
+
+          value than if prices were all available.
       borrowed_value:
         type: string
         description: >-
           Borrowed Value is the sum of the USD value of all tokens the account
           has borrowed, including interest owed.
 
           It always uses spot prices.
+
+          Computation skips borrows which are missing oracle prices, potentially
+          resulting in a lower borrowed
+
+          value than if prices were all available.
       borrow_limit:
         type: string
         description: >-
@@ -2043,11 +2081,18 @@ definitions:
 
           The lower of spot or historic price for each collateral token is used
           when calculating borrow limits.
+
+          Computation skips collateral which is missing an oracle price,
+          potentially resulting in a lower borrow
+
+          limit than if prices were all available.
       liquidation_threshold:
         type: string
         description: >-
           Liquidation Threshold is the Borrowed Value at which the account
           becomes eligible for liquidation.
+
+          Will be null if an oracle price required for computation is missing.
     description: >-
       QueryAccountSummaryResponse defines the response structure for the
       AccountSummary gRPC service handler.

x/leverage/client/tests/tests.go

@@ -252,6 +252,8 @@ func (s *IntegrationTestSuite) TestLeverageScenario() {
 		nil,
 	}
 
+	lt1 := sdk.MustNewDecFromStr("0.0085610525")
+
 	nonzeroQueries := []testQuery{
 		{
 			"query account balances",
@@ -294,7 +296,7 @@ func (s *IntegrationTestSuite) TestLeverageScenario() {
 				// (1001 / 1000000) * 34.21 * 0.25 = 0.0085610525
 				BorrowLimit: sdk.MustNewDecFromStr("0.0085610525"),
 				// (1001 / 1000000) * 0.25 * 34.21 = 0.0085610525
-				LiquidationThreshold: sdk.MustNewDecFromStr("0.0085610525"),
+				LiquidationThreshold: &lt1,
 			},
 		},
 		{

x/leverage/keeper/borrows.go

@@ -9,7 +9,9 @@ import (
 
 // assertBorrowerHealth returns an error if a borrower is currently above their borrow limit,
 // under either recent (historic median) or current prices. It returns an error if
-// prices cannot be calculated.
+// borrowed asset prices cannot be calculated, but will try to treat collateral whose prices are
+// unavailable as having zero value. This can still result in a borrow limit being too low,
+// unless the remaining collateral is enough to cover all borrows.
 // This should be checked in msg_server.go at the end of any transaction which is restricted
 // by borrow limits, i.e. Borrow, Decollateralize, Withdraw, MaxWithdraw.
 func (k Keeper) assertBorrowerHealth(ctx sdk.Context, borrowerAddr sdk.AccAddress) error {
@@ -20,8 +22,9 @@ func (k Keeper) assertBorrowerHealth(ctx sdk.Context, borrowerAddr sdk.AccAddres
 	if err != nil {
 		return err
 	}
-	limit, err := k.CalculateBorrowLimit(ctx, collateral)
+	limit, err := k.VisibleBorrowLimit(ctx, collateral)
 	if err != nil {
+		// This error is not a missing price
 		return err
 	}
 	if value.GT(limit) {
@@ -127,6 +130,43 @@ func (k Keeper) CalculateBorrowLimit(ctx sdk.Context, collateral sdk.Coins) (sdk
 	return limit, nil
 }
 
+// VisibleBorrowLimit uses the price oracle to determine the borrow limit (in USD) provided by
+// collateral sdk.Coins, using each token's uToken exchange rate and collateral weight.
+// The lower of spot price or historic price is used for each collateral token.
+// An error is returned if any input coins are not uTokens.
+// This function skips assets that are missing prices, which will lead to a lower borrow
+// limit when prices are down instead of a complete loss of borrowing ability.
+func (k Keeper) VisibleBorrowLimit(ctx sdk.Context, collateral sdk.Coins) (sdk.Dec, error) {
+	limit := sdk.ZeroDec()
+
+	for _, coin := range collateral {
+		// convert uToken collateral to base assets
+		baseAsset, err := k.ExchangeUToken(ctx, coin)
+		if err != nil {
+			return sdk.ZeroDec(), err
+		}
+
+		ts, err := k.GetTokenSettings(ctx, baseAsset.Denom)
+		if err != nil {
+			return sdk.ZeroDec(), err
+		}
+
+		// ignore blacklisted tokens
+		if !ts.Blacklist {
+			// get USD value of base assets using the chosen price mode
+			v, err := k.TokenValue(ctx, baseAsset, types.PriceModeLow)
+			if err == nil {
+				// if both spot and historic (if requeired) prices exist,
+				// add each collateral coin's weighted value to borrow limit
+				limit = limit.Add(v.Mul(ts.CollateralWeight))
+			}
+
+		}
+	}
+
+	return limit, nil
+}
+
 // CalculateLiquidationThreshold determines the maximum borrowed value (in USD) that a
 // borrower with given collateral could reach before being eligible for liquidation, using
 // each token's oracle price, uToken exchange rate, and liquidation threshold.

x/leverage/keeper/grpc_query.go

@@ -208,42 +208,45 @@ func (q Querier) AccountSummary(
 	collateral := q.Keeper.GetBorrowerCollateral(ctx, addr)
 	borrowed := q.Keeper.GetBorrowerBorrows(ctx, addr)
 
-	// supplied value always uses spot prices
-	suppliedValue, err := q.Keeper.TotalTokenValue(ctx, supplied, types.PriceModeSpot)
-	if err != nil {
-		return nil, err
-	}
-	// borrowed value here is shown using spot prices, but leverage logic instead uses
+	// supplied value always uses spot prices, and skips supplied assets that are missing prices
+	suppliedValue := q.Keeper.VisibleTokenValue(ctx, supplied, types.PriceModeSpot)
+
+	// borrowed value uses spot prices here, but leverage logic instead uses
 	// the higher of spot or historic prices for each borrowed token when comparing it
-	// to borrow limit.
-	borrowedValue, err := q.Keeper.TotalTokenValue(ctx, borrowed, types.PriceModeSpot)
-	if err != nil {
-		return nil, err
-	}
-	// collateral value always uses spot prices
-	collateralValue, err := q.Keeper.CalculateCollateralValue(ctx, collateral)
+	// to borrow limit. This line also skips borrowed assets that are missing prices.
+	borrowedValue := q.Keeper.VisibleTokenValue(ctx, borrowed, types.PriceModeSpot)
+
+	// collateral value always uses spot prices, and this line skips assets that are missing prices
+	collateralValue, err := q.Keeper.VisibleCollateralValue(ctx, collateral)
 	if err != nil {
+		// this error isn't a missing price - it would be non-uToken collateral
 		return nil, err
 	}
+
 	// borrow limit shown here as it is used in leverage logic:
 	// using the lower of spot or historic prices for each collateral token
-	borrowLimit, err := q.Keeper.CalculateBorrowLimit(ctx, collateral)
+	// skips collateral tokens with missing oracle prices
+	borrowLimit, err := q.Keeper.VisibleBorrowLimit(ctx, collateral)
 	if err != nil {
+		// this error isn't a missing price - it would be non-uToken collateral
 		return nil, err
 	}
-	// liquidation always uses spot prices
+
+	resp := &types.QueryAccountSummaryResponse{
+		SuppliedValue:   suppliedValue,
+		CollateralValue: collateralValue,
+		BorrowedValue:   borrowedValue,
+		BorrowLimit:     borrowLimit,
+	}
+
+	// liquidation always uses spot prices. This response field will be null
+	// if a price is missing
 	liquidationThreshold, err := q.Keeper.CalculateLiquidationThreshold(ctx, collateral)
-	if err != nil {
-		return nil, err
+	if err == nil {
+		resp.LiquidationThreshold = &liquidationThreshold
 	}
 
-	return &types.QueryAccountSummaryResponse{
-		SuppliedValue:        suppliedValue,
-		CollateralValue:      collateralValue,
-		BorrowedValue:        borrowedValue,
-		BorrowLimit:          borrowLimit,
-		LiquidationThreshold: liquidationThreshold,
-	}, nil
+	return resp, nil
 }
 
 func (q Querier) LiquidationTargets(
@@ -320,11 +323,13 @@ func (q Querier) MaxWithdraw(
 	}
 
 	for _, denom := range denoms {
+		// If a price is missing for the borrower's collateral,
+		// but not this uToken or any of their borrows, error
+		// will be nil and the resulting value will be what
+		// can safely be withdrawn even with missing prices.
+		// On non-nil error here, max withdraw is zero.
 		uToken, err := q.Keeper.maxWithdraw(ctx, addr, denom)
-		if err != nil {
-			return nil, err
-		}
-		if uToken.IsPositive() {
+		if err == nil && uToken.IsPositive() {
 			token, err := q.Keeper.ExchangeUToken(ctx, uToken)
 			if err != nil {
 				return nil, err
@@ -374,14 +379,15 @@ func (q Querier) MaxBorrow(
 	}
 
 	for _, denom := range denoms {
+		// If a price is missing for the borrower's collateral,
+		// but not this token or any of their borrows, error
+		// will be nil and the resulting value will be what
+		// can safely be borrowed even with missing prices.
+		// On non-nil error here, max borrow is zero.
 		maxBorrow, err := q.Keeper.maxBorrow(ctx, addr, denom)
-		if err != nil {
-			return nil, err
-		}
-		if maxBorrow.IsPositive() {
+		if err == nil && maxBorrow.IsPositive() {
 			maxTokens = maxTokens.Add(maxBorrow)
 		}
-
 	}
 
 	return &types.QueryMaxBorrowResponse{

x/leverage/keeper/grpc_query_test.go

@@ -98,6 +98,7 @@ func (s *IntegrationTestSuite) TestQuerier_AccountSummary() {
 	resp, err := s.queryClient.AccountSummary(ctx.Context(), &types.QueryAccountSummary{Address: addr.String()})
 	require.NoError(err)
 
+	lt := sdk.MustNewDecFromStr("1052.5")
 	expected := types.QueryAccountSummaryResponse{
 		// This result is umee's oracle exchange rate from
 		// from .Reset() in x/leverage/keeper/oracle_test.go
@@ -112,7 +113,7 @@ func (s *IntegrationTestSuite) TestQuerier_AccountSummary() {
 		// (1000) * 4.21 * 0.25 = 1052.5
 		BorrowLimit: sdk.MustNewDecFromStr("1052.5"),
 		// (1000) * 4.21 * 0.25 = 1052.5
-		LiquidationThreshold: sdk.MustNewDecFromStr("1052.5"),
+		LiquidationThreshold: &lt,
 	}
 
 	require.Equal(expected, *resp)

x/leverage/keeper/iter.go

@@ -168,20 +168,16 @@ func (k Keeper) GetEligibleLiquidationTargets(ctx sdk.Context) ([]sdk.AccAddress
 		collateral := k.GetBorrowerCollateral(ctx, addr)
 
 		// use oracle helper functions to find total borrowed value in USD
-		borrowValue, err := k.TotalTokenValue(ctx, borrowed, types.PriceModeSpot)
-		if err != nil {
-			return err
-		}
+		// skips denoms without prices
+		borrowValue := k.VisibleTokenValue(ctx, borrowed, types.PriceModeSpot)
 
 		// compute liquidation threshold from enabled collateral
+		// in this case, we can't reasonably skip missing prices but can move on
+		// to the next borrower instead of stopping the entire query
 		liquidationLimit, err := k.CalculateLiquidationThreshold(ctx, collateral)
-		if err != nil {
-			return err
-		}
-
-		// If liquidation limit is smaller than borrowed value then the
-		// address is eligible for liquidation.
-		if liquidationLimit.LT(borrowValue) {
+		if err == nil && liquidationLimit.LT(borrowValue) {
+			// If liquidation limit is smaller than borrowed value then the
+			// address is eligible for liquidation.
 			liquidationTargets = append(liquidationTargets, addr)
 		}