API: Experimental full assets for account endpoint.

ledger/catchpointfilewriter_test.go

@@ -685,7 +685,9 @@ func testNewLedgerFromCatchpoint(t *testing.T, catchpointWriterReadAccess tracke
 	var initState ledgercore.InitState
 	initState.Block.CurrentProtocol = protocol.ConsensusCurrentVersion
 	conf := config.GetDefaultLocal()
-	l, err := OpenLedger(logging.TestingLog(t), t.Name()+"FromCatchpoint", true, initState, conf)
+	dbName := fmt.Sprintf("%s.%d", t.Name()+"FromCatchpoint", crypto.RandUint64())
+	dbName = strings.Replace(dbName, "/", "_", -1)
+	l, err := OpenLedger(logging.TestingLog(t), dbName, true, initState, conf)
 	require.NoError(t, err)
 	accessor := MakeCatchpointCatchupAccessor(l, l.log)
 
@@ -702,6 +704,8 @@ func testNewLedgerFromCatchpoint(t *testing.T, catchpointWriterReadAccess tracke
 	err = accessor.BuildMerkleTrie(context.Background(), nil)
 	require.NoError(t, err)
 
+	resetAccountDBToV6(t, l)
+
 	err = l.trackerDBs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
 		cw, err := tx.MakeCatchpointWriter()
 		if err != nil {

ledger/ledger_test.go

@@ -53,6 +53,8 @@ import (
 	"github.com/algorand/go-deadlock"
 )
 
+const preReleaseDBVersion = 6
+
 func sign(secrets map[basics.Address]*crypto.SignatureSecrets, t transactions.Transaction) transactions.SignedTxn {
 	var sig crypto.Signature
 	_, ok := secrets[t.Sender]
@@ -2178,6 +2180,42 @@ func TestLedgerReloadShrinkDeltas(t *testing.T) {
 	}
 }
 
+func resetAccountDBToV6(t *testing.T, l *Ledger) {
+	// reset tables and re-init again, similary to the catchpount apply code
+	// since the ledger has only genesis accounts, this recreates them
+	err := l.trackerDBs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
+		arw, err := tx.MakeAccountsWriter()
+		if err != nil {
+			return err
+		}
+
+		err0 := arw.AccountsReset(ctx)
+		if err0 != nil {
+			return err0
+		}
+		tp := trackerdb.Params{
+			InitAccounts:      l.GenesisAccounts(),
+			InitProto:         l.GenesisProtoVersion(),
+			GenesisHash:       l.GenesisHash(),
+			FromCatchpoint:    true,
+			CatchpointEnabled: l.catchpoint.catchpointEnabled(),
+			DbPathPrefix:      l.catchpoint.dbDirectory,
+			BlockDb:           l.blockDBs,
+		}
+		_, err0 = tx.RunMigrations(ctx, tp, l.log, preReleaseDBVersion /*target database version*/)
+		if err0 != nil {
+			return err0
+		}
+
+		if err0 := tx.Testing().AccountsUpdateSchemaTest(ctx); err != nil {
+			return err0
+		}
+
+		return nil
+	})
+	require.NoError(t, err)
+}
+
 // TestLedgerReloadTxTailHistoryAccess checks txtail has MaxTxnLife + DeeperBlockHeaderHistory block headers
 // for TEAL after applying catchpoint.
 // Simulate catchpoints by the following:
@@ -2189,8 +2227,6 @@ func TestLedgerReloadShrinkDeltas(t *testing.T) {
 func TestLedgerReloadTxTailHistoryAccess(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	const preReleaseDBVersion = 6
-
 	dbName := fmt.Sprintf("%s.%d", t.Name(), crypto.RandUint64())
 	genesisInitState, initKeys := ledgertesting.GenerateInitState(t, protocol.ConsensusCurrentVersion, 10_000_000_000)
 	proto := config.Consensus[protocol.ConsensusCurrentVersion]

ledger/store/trackerdb/sqlitedriver/schema.go

@@ -943,78 +943,83 @@
 	return err
 }
 
-func accountsAddCreatableTypeColumn(ctx context.Context, e db.Executable) error {
+func accountsAddCreatableTypeColumn(ctx context.Context, e db.Executable, populateColumn bool) error {
 	// Run ctype resources migration if it hasn't run yet
 	var creatableTypeOnResourcesRun bool
 	err := e.QueryRow("SELECT 1 FROM pragma_table_info('resources') WHERE name='ctype'").Scan(&creatableTypeOnResourcesRun)
-	if !errors.Is(err, sql.ErrNoRows) {
+	if err == nil {
+		// Already exists.
 		return nil
-	}
+	} else if !errors.Is(err, sql.ErrNoRows) {
+		return err
+	} // A sql.ErrNoRows error means the column does not exist, so we need to create it
 
 	// Add ctype column
 	createStmt := `ALTER TABLE resources ADD COLUMN ctype INTEGER NOT NULL DEFAULT -1`
 
 	_, err = e.ExecContext(ctx, createStmt)
 	if err != nil {
 		return err
 	}
 
-	// Populate the new ctype column with the corresponding creatable type from assetcreators where available
-	updateStmt := `UPDATE resources SET ctype = (
+	if populateColumn {
+		// Populate the new ctype column with the corresponding creatable type from assetcreators where available
+		updateStmt := `UPDATE resources SET ctype = (
     SELECT COALESCE((SELECT ac.ctype FROM assetcreators ac WHERE ac.asset = resources.aidx),-1)
 	) WHERE ctype = -1`
 
-	_, err = e.ExecContext(ctx, updateStmt)
-	if err != nil {
-		return err
-	}
-
-	updatePrepStmt, err := e.PrepareContext(ctx, "UPDATE resources SET ctype = ? WHERE addrid = ? AND aidx = ?")
-	if err != nil {
-		return err
-	}
-	defer updatePrepStmt.Close()
-
-	// Pull resource entries into memory where ctype is not set
-	rows, err := e.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources r WHERE ctype = -1")
-	if err != nil {
-		return err
-	}
-	defer rows.Close()
-
-	// Update the ctype column for subset of resources where ctype was not resolved from assetcreators
-	for rows.Next() {
-		var addrid int64
-		var aidx int64
-		var encodedData []byte
-		err = rows.Scan(&addrid, &aidx, &encodedData)
-		if err != nil {
-			return err
+		_, err0 := e.ExecContext(ctx, updateStmt)
+		if err0 != nil {
+			return err0
 		}
 
-		var rd trackerdb.ResourcesData
-		err = protocol.Decode(encodedData, &rd)
-		if err != nil {
-			return err
+		updatePrepStmt, err0 := e.PrepareContext(ctx, "UPDATE resources SET ctype = ? WHERE addrid = ? AND aidx = ?")
+		if err0 != nil {
+			return err0
 		}
+		defer updatePrepStmt.Close()
 
-		var ct basics.CreatableType
-		if rd.IsAsset() && rd.IsApp() {
-			// This should never happen!
-			return fmt.Errorf("unable to discern creatable type for addrid %d, resource %d", addrid, aidx)
-		} else if rd.IsAsset() {
-			ct = basics.AssetCreatable
-		} else if rd.IsApp() {
-			ct = basics.AppCreatable
-		} else { // This should never happen!
-			return fmt.Errorf("unable to discern creatable type for addrid %d, resource %d", addrid, aidx)
+		// Pull resource entries into memory where ctype is not set
+		rows, err0 := e.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources r WHERE ctype = -1")
+		if err0 != nil {
+			return err0
 		}
+		defer rows.Close()
 
-		_, err = updatePrepStmt.ExecContext(ctx, ct, addrid, aidx)
-		if err != nil {
-			return err
+		// Update the ctype column for subset of resources where ctype was not resolved from assetcreators
+		for rows.Next() {
+			var addrid int64
+			var aidx int64
+			var encodedData []byte
+			err0 = rows.Scan(&addrid, &aidx, &encodedData)
+			if err0 != nil {
+				return err0
+			}
+
+			var rd trackerdb.ResourcesData
+			err0 = protocol.Decode(encodedData, &rd)
+			if err0 != nil {
+				return err0
+			}
+
+			var ct basics.CreatableType
+			if rd.IsAsset() && rd.IsApp() {
+				// This should never happen!
+				return fmt.Errorf("unable to discern creatable type for addrid %d, resource %d", addrid, aidx)
+			} else if rd.IsAsset() {
+				ct = basics.AssetCreatable
+			} else if rd.IsApp() {
+				ct = basics.AppCreatable
+			} else { // This should never happen!
+				return fmt.Errorf("unable to discern creatable type for addrid %d, resource %d", addrid, aidx)
+			}
+
+			_, err0 = updatePrepStmt.ExecContext(ctx, ct, addrid, aidx)
+			if err0 != nil {
+				return err0
+			}
 		}
 	}
 
-	return rows.Err()
+	return nil
 }

ledger/store/trackerdb/sqlitedriver/testing.go

@@ -101,29 +101,37 @@
 	err = performKVStoreNullBlobConversion(context.Background(), e)
 	require.NoError(tb, err)
 
-	err = accountsAddCreatableTypeColumn(context.Background(), e)
+	err = accountsAddCreatableTypeColumn(context.Background(), e, false)
 	require.NoError(tb, err)
 
 	return newDB
 }
 
 // AccountsUpdateSchemaTest adds some empty tables for tests to work with a "v6" store.
 func AccountsUpdateSchemaTest(ctx context.Context, e db.Executable) (err error) {
-	if err := accountsCreateOnlineAccountsTable(ctx, e); err != nil {
+	if err = accountsCreateOnlineAccountsTable(ctx, e); err != nil {
 		return err
 	}
-	if err := accountsCreateTxTailTable(ctx, e); err != nil {
+	if err = accountsCreateTxTailTable(ctx, e); err != nil {
 		return err
 	}
-	if err := accountsCreateOnlineRoundParamsTable(ctx, e); err != nil {
+	if err = accountsCreateOnlineRoundParamsTable(ctx, e); err != nil {
 		return err
 	}
-	if err := accountsCreateCatchpointFirstStageInfoTable(ctx, e); err != nil {
+	if err = accountsCreateCatchpointFirstStageInfoTable(ctx, e); err != nil {
 		return err
 	}
 	// this line creates kvstore table, even if it is not required in accountDBVersion 6 -> 7
 	// or in later version where we need kvstore table, some tests will fail
-	if err := accountsCreateBoxTable(ctx, e); err != nil {
+	if err = accountsCreateBoxTable(ctx, e); err != nil {
+		return err
+	}
+	// this adds the resources table and ctype column, even if it is not required in accountDBVersion 6 -> 7
+	// this prevents some tests from failing.
+	if err = accountsCreateResourceTable(ctx, e); err != nil {
+		return err
+	}
+	if err = accountsAddCreatableTypeColumn(ctx, e, false); err != nil {
 		return err
 	}
 	return createStateProofVerificationTable(ctx, e)

ledger/store/trackerdb/sqlitedriver/trackerdbV2.go

@@ -136,11 +136,11 @@
 					tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 9 : %v", err)
 					return
 				}
 			case 10:
 				err = tu.upgradeDatabaseSchema10(ctx, e)
 				if err != nil {
 					tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 10 : %v", err)
 					return
 				}
 			default:
 				return trackerdb.InitParams{}, fmt.Errorf("trackerDBInitialize unable to upgrade database from schema version %d", tu.schemaVersion)
@@ -515,13 +515,13 @@
 
 // upgradeDatabaseSchema10 upgrades the database schema from version 10 to version 11,
 // altering the resources table to add a new column, resources.ctype.
 func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema10(ctx context.Context, e db.Executable) (err error) {
-	err = accountsAddCreatableTypeColumn(ctx, e)
+	err = accountsAddCreatableTypeColumn(ctx, e, true)
 	if err != nil {
 		return err
 	}
 	// update version
 	return tu.setVersion(ctx, e, 11)
 }
 
 func removeEmptyDirsOnSchemaUpgrade(dbDirectory string) (err error) {

ledger/store/trackerdb/testsuite/accounts_kv_test.go

@@ -290,9 +290,9 @@ func CustomTestResourcesQueryAllLimited(t *customT) {
 	resDataA1AcctA := trackerdb.ResourcesData{}
 	appParams := ledgertesting.RandomAppParams()
 	resDataA1AcctA.SetAppParams(appParams, true)
+	resDataA1AcctA.SetAppLocalState(basics.AppLocalState{})
 	resDataA1AcctB := trackerdb.ResourcesData{}
-	appParamsB := ledgertesting.RandomAppParams()
-	resDataA1AcctB.SetAppParams(appParamsB, true)
+	resDataA1AcctB.SetAppLocalState(basics.AppLocalState{})
 	aidxResA1 := basics.CreatableIndex(2)
 	_, err = aow.InsertResource(refAccA, aidxResA1, resDataA1AcctA)
 	require.NoError(t, err)
@@ -383,6 +383,10 @@ func CustomTestResourcesQueryAllLimited(t *customT) {
 	require.Equal(t, resDataWithParamsA0AcctB, prs[0].Data)
 	require.Equal(t, expectedRound, rnd) // db round (from the return)
 
+	// Delete app owner for A-1
+	_, err = aow.DeleteCreatable(aidxResA1, basics.AppCreatable)
+	require.NoError(t, err)
+
 	// Set min to 1, should return only 1 resource (index 1)
 	prs, rnd, err = aor.LookupLimitedResources(addrB, 1, 1, basics.AssetCreatable)
 	require.NoError(t, err)