fix: token transfer columns

models/contrib/debt-dao/generalized_cashflow.py

@@ -18,7 +18,7 @@ class Config:
 
 @Model.describe(
     slug='contrib.debt-dao-generalized-cashflow',
-    version='1.2',
+    version='1.3',
     display_name='Generalized Cashflow',
     description='Tracks cashflow from sender address to receiver address.',
     category='protocol',
@@ -30,8 +30,8 @@ class GeneralizedCashflow(Model):
     def run(self, input: GCInput) -> dict:
         with self.context.ledger.TokenTransfer as q:
             transfers = q.select(
+                aggregates=[(q.RAW_AMOUNT, 'value')],
                 columns=[q.BLOCK_NUMBER,
-                         q.VALUE,
                          q.TOKEN_ADDRESS,
                          q.TRANSACTION_HASH],
                 where=q.TO_ADDRESS.eq(input.receiver_address).and_(
@@ -48,7 +48,9 @@ def run(self, input: GCInput) -> dict:
                 transfer['price'] = 0
             if transfer['price'] is None:
                 transfer['price'] = 0
-            transfer['value_usd'] = transfer['price'] * float(transfer['value']) / (10 ** token.decimals)
-            transfer['block_time'] = str(BlockNumber(int(transfer['block_number'])).timestamp_datetime)
+            transfer['value_usd'] = transfer['price'] * \
+                float(transfer['value']) / (10 ** token.decimals)
+            transfer['block_time'] = str(BlockNumber(
+                int(transfer['block_number'])).timestamp_datetime)
             transfer['token_symbol'] = token.symbol
         return transfers.dict()

models/contrib/exa/token.py

@@ -54,7 +54,7 @@ def run(self, input: TokenBalanceInput):
                 description=("return net token outflow from an address on a given time range"
                              "net inflow is total token inflow - total token outflow"
                              ),
-                version='1.1',
+                version='1.2',
                 developer='exa256',
                 input=TokenNetInflowInput,
                 output=dict
@@ -72,7 +72,7 @@ def run(self, input: TokenNetInflowInput):
                 columns=[
                     q.TO_ADDRESS,
                     q.FROM_ADDRESS,
-                    q.VALUE,
+                    q.RAW_AMOUNT,
                 ],
                 where=q.BLOCK_NUMBER.gt(past_block).and_(
                     q.TOKEN_ADDRESS.eq(token.address).and_(
@@ -85,9 +85,9 @@ def run(self, input: TokenNetInflowInput):
             ).to_dataframe()
 
         inflow = transfers.query(
-            'to_address == @from_addr')['value'].astype(float).sum()
+            'to_address == @from_addr')['raw_amount'].astype(float).sum()
         outflow = transfers.query(
-            'from_address == @from_addr')['value'].astype(float).sum()
+            'from_address == @from_addr')['raw_amount'].astype(float).sum()
 
         return {
             'inflow': token.scaled(inflow),

models/contrib/kunlun/uniswap_fee.py

@@ -48,7 +48,7 @@ def default(cls, input: UniswapFeeInput):
 
 
 @Model.describe(slug='contrib.uniswap-fee',
-                version='1.2',
+                version='1.3',
                 display_name='Calculate fee from swaps in Uniswap V3 pool',
                 description="Ledger",
                 input=UniswapFeeInput,
@@ -83,7 +83,7 @@ def run(self, input: UniswapFeeInput) -> UniswapFeeOutput:
                 df_tt = q.select(
                     columns=q_cols,
                     aggregates=[((f'CASE WHEN {q.TO_ADDRESS.eq(uni_pool_addr)} '
-                                  f'THEN {q.VALUE} ELSE {q.VALUE.neg_()} END'),
+                                  f'THEN {q.RAW_AMOUNT} ELSE {q.RAW_AMOUNT.neg_()} END'),
                                  'transaction_value')],
                     where=(q.BLOCK_NUMBER.gt(block_start).and_(q.BLOCK_NUMBER.le(block_end))
                            .and_(q.FROM_ADDRESS.eq(uni_pool_addr)
@@ -183,7 +183,8 @@ def calculate_fee(r, models, t0, t1, fee):
 
         df_new_cols = df_tx_swap_one_line.apply(
             lambda r, self=self, t0=t0, t1=t1, fee=fee:
-            calculate_fee(r, self.context.models, t0, t1, fee), axis=1, result_type='expand')  # type: ignore
+            # type: ignore
+            calculate_fee(r, self.context.models, t0, t1, fee), axis=1, result_type='expand')
 
         df_new_cols.columns = pd.Index(['in_value', 'out_value', 'fee'])
 

models/contrib/neilz/research.py

@@ -1,19 +1,12 @@
 # pylint: disable=line-too-long
 
 from credmark.cmf.model import Model
-from credmark.cmf.types import (
-    Address,
-    BlockNumber,
-    MapBlocksOutput,
-    Maybe,
-    PriceWithQuote,
-    Token,
-)
+from credmark.cmf.types import Address, BlockNumber, MapBlocksOutput, Maybe, PriceWithQuote, Token
 from credmark.dto import EmptyInputSkipTest
 
 
 @Model.describe(slug='contrib.neilz-redacted-votium-cashflow',
-                version='1.2',
+                version='1.3',
                 display_name='Redacted Cartel Votium Cashflow',
                 description='Redacted Cartel Votium Cashflow',
                 category='protocol',
@@ -28,13 +21,15 @@ def run(self, _) -> dict:
         redacted_multisig_address = Address(
             "0xA52Fd396891E7A74b641a2Cb1A6999Fcf56B077e")
         with self.context.ledger.TokenTransfer as q:
-            transfers = q.select(columns=[
-                q.BLOCK_NUMBER,
-                q.VALUE,
-                q.TOKEN_ADDRESS,
-                q.TRANSACTION_HASH
-            ], where=q.TO_ADDRESS.eq(redacted_multisig_address).and_(
-                q.FROM_ADDRESS.eq(votium_claim_address)))
+            transfers = q.select(
+                aggregates=[(q.RAW_AMOUNT, 'value')],
+                columns=[
+                    q.BLOCK_NUMBER,
+                    q.RAW_AMOUNT,
+                    q.TOKEN_ADDRESS,
+                    q.TRANSACTION_HASH
+                ], where=q.TO_ADDRESS.eq(redacted_multisig_address).and_(
+                    q.FROM_ADDRESS.eq(votium_claim_address)))
 
         for transfer in transfers:
             transfer['block_number'] = int(transfer['block_number'])
@@ -48,15 +43,16 @@ def run(self, _) -> dict:
                 transfer['price'] = 0
             if transfer['price'] is None:
                 transfer['price'] = 0
-            transfer['value_usd'] = transfer['price'] * float(transfer['value']) / (10 ** token.decimals)
+            transfer['value_usd'] = transfer['price'] * \
+                float(transfer['value']) / (10 ** token.decimals)
             transfer['block_time'] = str(BlockNumber(transfer['block_number'])
                                          .timestamp_datetime)
             transfer['token_symbol'] = token.symbol
         return transfers.dict()
 
 
 @Model.describe(slug='contrib.neilz-redacted-convex-cashflow',
-                version='1.4',
+                version='1.5',
                 display_name='Redacted Cartel Convex Cashflow',
                 description='Redacted Cartel Convex Cashflow',
                 category='protocol',
@@ -73,12 +69,14 @@ class RedactedConvexCashflow(Model):
 
     def run(self, _) -> dict:
         with self.context.ledger.TokenTransfer as q:
-            transfers = q.select(columns=[
-                q.BLOCK_NUMBER,
-                q.VALUE,
-                q.TOKEN_ADDRESS
-            ], where=q.TO_ADDRESS.eq(self.REDACTED_MULTISIG_ADDRESS).and_(
-                q.FROM_ADDRESS.in_(self.CONVEX_ADDRESSES))
+            transfers = q.select(
+                aggregates=[(q.RAW_AMOUNT, 'value')],
+                columns=[
+                    q.BLOCK_NUMBER,
+                    q.RAW_AMOUNT,
+                    q.TOKEN_ADDRESS
+                ], where=q.TO_ADDRESS.eq(self.REDACTED_MULTISIG_ADDRESS).and_(
+                    q.FROM_ADDRESS.in_(self.CONVEX_ADDRESSES))
             )
 
         token_prices = {}

models/credmark/ledger/transfers.py

@@ -93,42 +93,6 @@ def get_token_transfer(_context,
                        start_block: int,
                        fix_int: bool = True,
                        limit: Optional[int] = None) -> pd.DataFrame:
-    def _use_ledger():
-        with _context.ledger.TokenTransfer as q:
-            transfer_cols = [q.BLOCK_NUMBER, q.TO_ADDRESS, q.FROM_ADDRESS, q.TOKEN_ADDRESS,
-                             q.TRANSACTION_HASH, q.LOG_INDEX]
-            df_ts = [pd.DataFrame(columns=transfer_cols+['value'], data=[])]
-
-        with _context.ledger.TokenTransfer as q:
-            for _address in _accounts:
-                where_cond = (q.TO_ADDRESS.eq(_address).or_(
-                    q.FROM_ADDRESS.eq(_address))).parentheses_()
-                if len(_tokens) > 0:
-                    where_cond = where_cond.and_(q.TOKEN_ADDRESS.in_(_tokens))
-                if start_block > 0:
-                    where_cond = where_cond.and_(
-                        q.BLOCK_NUMBER.le(start_block))
-                offset = 0
-                while True:
-                    df_tt = (q.select(
-                        columns=transfer_cols,
-                        aggregates=[((f'CASE WHEN {q.TO_ADDRESS.eq(_address)} '
-                                      f'THEN {q.VALUE} ELSE {q.VALUE.neg_()} END'), 'value')],
-                        where=where_cond,
-                        order_by=q.BLOCK_NUMBER.comma_(q.LOG_INDEX),
-                        offset=offset).to_dataframe())
-
-                    if df_tt.shape[0] > 0:
-                        df_ts.append(df_tt)
-                    if df_tt.shape[0] < 5000:
-                        break
-                    offset += 5000
-
-        return (pd.concat(df_ts)
-                .assign(block_number=lambda x: x.block_number.apply(int))
-                .drop_duplicates()
-                .sort_values('block_number')
-                .reset_index(drop=True))
 
     def _use_model():
         req = {'accounts': _accounts, 'startBlock': start_block}

models/credmark/tokens/netflow.py

@@ -3,14 +3,7 @@
 
 from credmark.cmf.model import Model
 from credmark.cmf.model.errors import ModelRunError
-from credmark.cmf.types import (
-    Address,
-    BlockNumber,
-    JoinType,
-    NativeToken,
-    PriceWithQuote,
-    Token,
-)
+from credmark.cmf.types import Address, BlockNumber, JoinType, NativeToken, PriceWithQuote, Token
 from credmark.dto import DTO, DTOField
 
 
@@ -64,7 +57,7 @@ class TokenNetflowOutput(TokenNetflowBlockRange):
 
 
 @Model.describe(slug='token.netflow-block',
-                version='1.4',
+                version='1.5',
                 display_name='Token netflow',
                 description='The Current Credmark Supported netflow algorithm',
                 category='protocol',
@@ -108,11 +101,11 @@ def run(self, input: TokenNetflowBlockInput) -> TokenNetflowOutput:
                 df = q.select(
                     aggregates=[
                         ((f'SUM(CASE WHEN {q.TO_ADDRESS.eq(input.netflow_address)} '
-                          f'THEN {q.VALUE} ELSE 0::INTEGER END)'), 'inflow'),
+                          f'THEN {q.RAW_AMOUNT} ELSE 0::INTEGER END)'), 'inflow'),
                         ((f'SUM(CASE WHEN {q.FROM_ADDRESS.eq(input.netflow_address)} '
-                          f'THEN {q.VALUE} ELSE 0::INTEGER END)'), 'outflow'),
+                          f'THEN {q.RAW_AMOUNT} ELSE 0::INTEGER END)'), 'outflow'),
                         ((f'SUM(CASE WHEN {q.TO_ADDRESS.eq(input.netflow_address)} '
-                          f'THEN {q.VALUE} ELSE {q.VALUE.neg_()} END)'), 'netflow')],
+                          f'THEN {q.RAW_AMOUNT} ELSE {q.RAW_AMOUNT.neg_()} END)'), 'netflow')],
                     where=q.BLOCK_NUMBER.gt(old_block)
                     .and_(q.TOKEN_ADDRESS.eq(token_address))
                     .and_(q.TO_ADDRESS.eq(input.netflow_address)
@@ -224,7 +217,7 @@ class TokenNetflowSegmentOutput(DTO):
 
 
 @Model.describe(slug='token.netflow-segment-block',
-                version='1.4',
+                version='1.5',
                 display_name='Token netflow by segment by block',
                 description='The Current Credmark Supported netflow algorithm',
                 category='protocol',
@@ -270,8 +263,8 @@ def run(self, input: TokenNetflowSegmentBlockInput) -> TokenNetflowSegmentOutput
         native_token = NativeToken()
         if token_address == native_token.address:
             input_token = native_token
-            with self.context.ledger.Transaction.as_('t') as t,\
-                    self.context.ledger.Block.as_('s') as s,\
+            with self.context.ledger.Transaction.as_('t') as t, \
+                    self.context.ledger.Block.as_('s') as s, \
                     self.context.ledger.Block.as_('e') as e:
 
                 df = s.select(
@@ -304,8 +297,8 @@ def run(self, input: TokenNetflowSegmentBlockInput) -> TokenNetflowSegmentOutput
                 from_iso8601_str = t.field('').from_iso8601_str
         else:
             input_token = input.token
-            with self.context.ledger.TokenTransfer.as_('t') as t,\
-                    self.context.ledger.Block.as_('s') as s,\
+            with self.context.ledger.TokenTransfer.as_('t') as t, \
+                    self.context.ledger.Block.as_('s') as s, \
                     self.context.ledger.Block.as_('e') as e:
 
                 df = s.select(
@@ -315,11 +308,11 @@ def run(self, input: TokenNetflowSegmentBlockInput) -> TokenNetflowSegmentOutput
                         (e.NUMBER, 'to_block'),
                         (e.TIMESTAMP, 'to_timestamp'),
                         ((f'SUM(CASE WHEN {t.TO_ADDRESS.eq(input.netflow_address)} '
-                          f'THEN {t.VALUE} ELSE 0::INTEGER END)'), 'inflow'),
+                          f'THEN {t.RAW_AMOUNT} ELSE 0::INTEGER END)'), 'inflow'),
                         ((f'SUM(CASE WHEN {t.FROM_ADDRESS.eq(input.netflow_address)} '
-                          f'THEN {t.VALUE} ELSE 0::INTEGER END)'), 'outflow'),
+                          f'THEN {t.RAW_AMOUNT} ELSE 0::INTEGER END)'), 'outflow'),
                         ((f'SUM(CASE WHEN {t.TO_ADDRESS.eq(input.netflow_address)} '
-                          f'THEN {t.VALUE} ELSE {t.VALUE.neg_()} END)'), 'netflow')
+                          f'THEN {t.RAW_AMOUNT} ELSE {t.RAW_AMOUNT.neg_()} END)'), 'netflow')
                     ],
                     joins=[
                         (e, e.NUMBER.eq(s.NUMBER.plus_(str(block_seg)).minus_(str(1)))),

models/credmark/tokens/transaction.py

@@ -172,7 +172,7 @@ def plot_dig(dig: nx.DiGraph, figsize=(7, 7)):
 # credmark-dev run token.transaction -i '{"hash": "0x319552805d5f3d0c97e7b6c1e40d0c42817c49406fbff41af0f3ac88b590aa34", "block_number": 15125867}'
 
 @Model.describe(slug='token.transaction',
-                version='0.2',
+                version='0.3',
                 display_name='Token Transaction',
                 description='Tagged transactions for token transfer',
                 developer='Credmark',
@@ -198,9 +198,11 @@ def run(self, input: TransactionTagInput) -> dict:
             return self.context.run_model(self.slug, input=input, block_number=input_block_number)
 
         with self.context.ledger.TokenTransfer as q:
-            df_txn = q.select(columns=q.columns,
-                              where=q.TRANSACTION_HASH.eq(input.hash).and_(
-                                  q.BLOCK_NUMBER.eq(input_block_number))).to_dataframe()
+            df_txn = q.select(
+                aggregates=[(q.RAW_AMOUNT, 'value')],
+                columns=q.columns,
+                where=q.TRANSACTION_HASH.eq(input.hash).and_(
+                    q.BLOCK_NUMBER.eq(input_block_number))).to_dataframe()
 
         return self.context.run_model('token.txn-classify', input=Records.from_dataframe(df_txn))