From 27b0bf5ca5117fd88d31240cb28fec1c0685ddf0 Mon Sep 17 00:00:00 2001 From: andrew <44451818+afostr@users.noreply.github.com> Date: Tue, 5 Aug 2025 08:40:17 -0500 Subject: [PATCH 1/3] patcher crash fix --- src/state-manager/AccountPatcher.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/state-manager/AccountPatcher.ts b/src/state-manager/AccountPatcher.ts index 6171aeea5..f476f8680 100644 --- a/src/state-manager/AccountPatcher.ts +++ b/src/state-manager/AccountPatcher.ts @@ -2772,7 +2772,7 @@ class AccountPatcher { // for non-wrapped ranges if (startRadix <= endRadix) { - for (let i = 0; i <= isInsyncResult.radixes.length; i++) { + for (let i = 0; i < isInsyncResult.radixes.length; i++) { const radixEntry = isInsyncResult.radixes[i] if (radixEntry.radix >= startRadix && radixEntry.radix <= endRadix) { radixEntry.recentRuntimeSync = true @@ -2781,7 +2781,7 @@ class AccountPatcher { } // for wrapped ranges because we start at the end and wrap around to the beginning of 32 byte address space } else { - for (let i = 0; i <= isInsyncResult.radixes.length; i++) { + for (let i = 0; i < isInsyncResult.radixes.length; i++) { const radixEntry = isInsyncResult.radixes[i] if (radixEntry.radix >= startRadix || radixEntry.radix <= endRadix) { radixEntry.recentRuntimeSync = true From 98021b19664297563bc6c30b7b7d63bf00c34f07 Mon Sep 17 00:00:00 2001 From: Sonali Thakur Date: Tue, 5 Aug 2025 17:46:28 +0530 Subject: [PATCH 2/3] ntroduced `debugFailToCommit` flag to allow intentional skipping of transaction commits for debugging purposes. formatting in StateManager and TransactionQueue --- src/state-manager/TransactionQueue.ts | 6 ++++++ src/state-manager/index.ts | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/state-manager/TransactionQueue.ts b/src/state-manager/TransactionQueue.ts index 8b2371b82..4ead0fd57 100644 --- a/src/state-manager/TransactionQueue.ts +++ b/src/state-manager/TransactionQueue.ts @@ -1447,6 +1447,12 @@ class TransactionQueue { * @param queueEntry */ async commitConsensedTransaction(queueEntry: QueueEntry): Promise { + // Debug flag to intentionally skip committing account data + if (this.stateManager.debugFailToCommit) { + /* prettier-ignore */ if (logFlags.debug) this.mainLogger.debug(`debugFailToCommit active. Skipping commit for tx: ${queueEntry.logID}`) + queueEntry.accountDataSet = true + return { success: true } + } let ourLockID = -1 let accountDataList: string | unknown[] let uniqueKeys = [] diff --git a/src/state-manager/index.ts b/src/state-manager/index.ts index 80522ad7c..11c4b6f51 100644 --- a/src/state-manager/index.ts +++ b/src/state-manager/index.ts @@ -183,6 +183,7 @@ class StateManager { debugNoTxVoting: boolean debugSkipPatcherRepair: boolean + debugFailToCommit: boolean ignoreRecieptChance: number ignoreVoteChance: number @@ -340,6 +341,7 @@ class StateManager { this.processCycleSummaries = false //starts false and get enabled when startProcessingCycleSummaries() is called this.debugSkipPatcherRepair = config.debug.skipPatcherRepair + this.debugFailToCommit = false this.feature_receiptMapResults = true this.feature_partitionHashes = true @@ -1291,6 +1293,16 @@ class StateManager { this.partitionStats.setupHandlers() + // Debug endpoint to toggle fail-to-commit flag + Context.network.registerExternalGet('debugFailToCommit', isDebugModeMiddleware, (req, res) => { + const { enable } = req.query + if (enable !== undefined) { + const val = enable === 'true' || enable === '1' + this.debugFailToCommit = val + } + res.json({ debugFailToCommit: this.debugFailToCommit }) + }) + // p2p ASK // this.p2p.registerInternal( // 'request_receipt_for_tx_old', From 04c20794d895158bcba60bc2d9a53ed33333a863 Mon Sep 17 00:00:00 2001 From: Sonali Thakur Date: Tue, 12 Aug 2025 13:13:47 +0530 Subject: [PATCH 3/3] Add bypassAccountCache option to ServerConfiguration and update AccountCache methods --- src/config/server.ts | 1 + src/shardus/shardus-types.ts | 2 + src/state-manager/AccountCache.ts | 20 +- src/state-manager/AccountGlobals.ts | 6 +- src/state-manager/AccountPatcher.ts | 43 +- src/state-manager/PartitionStats.ts | 16 +- src/state-manager/TransactionQueue.ts | 12 +- src/state-manager/TransactionRepair.ts | 14 +- src/state-manager/index.ts | 8 +- .../src/state-manager/AccountCache.test.ts | 442 ++++++++++++++++++ 10 files changed, 508 insertions(+), 56 deletions(-) create mode 100644 test/unit/src/state-manager/AccountCache.test.ts diff --git a/src/config/server.ts b/src/config/server.ts index e95e4fac7..2dd1baa2f 100644 --- a/src/config/server.ts +++ b/src/config/server.ts @@ -373,6 +373,7 @@ const SERVER_CONFIG: StrictServerConfiguration = { avoidOurIndexInFactTell: false, //initial testing shows this may cause issues so leaving it off for now checkDestLimits: true, checkDestLimitCount: 5, + bypassAccountCache: false, }, sharding: { nodesPerConsensusGroup: 5, nodesPerEdge: 2, executeInOneShard: false }, mode: ServerMode.Release, diff --git a/src/shardus/shardus-types.ts b/src/shardus/shardus-types.ts index bced52a81..730f41041 100644 --- a/src/shardus/shardus-types.ts +++ b/src/shardus/shardus-types.ts @@ -1410,6 +1410,8 @@ export interface ServerConfiguration { checkDestLimits: boolean // how many times can this destination address show up in the queue before we avoid sending to it checkDestLimitCount: number + // bypass account cache and load account data directly from storage + bypassAccountCache: boolean } /** Options for sharding calculations */ sharding?: { diff --git a/src/state-manager/AccountCache.ts b/src/state-manager/AccountCache.ts index 843e2c007..d322726dd 100644 --- a/src/state-manager/AccountCache.ts +++ b/src/state-manager/AccountCache.ts @@ -255,12 +255,28 @@ class AccountCache { // at the end of buildPartitionHashesForNode gets set to the working/current cycle. // if TXs come in that are newer they get put in the future list and are not part of the parition hash report yet - hasAccount(accountId: string): boolean { + async hasAccount(accountId: string): Promise { + if (this.config.stateManager.bypassAccountCache) { + const accountDataList = await this.app.getAccountDataByList([accountId]) + return !!(accountDataList && accountDataList.length > 0 && accountDataList[0] != null) + } return this.accountsHashCache3.accountHashMap.has(accountId) } //just gets the newest seen hash. does that cause issues? - getAccountHash(accountId: string): AccountHashCache { + async getAccountHash(accountId: string): Promise { + if (this.config.stateManager.bypassAccountCache) { + const accountDataList = await this.app.getAccountDataByList([accountId]) + if (accountDataList && accountDataList.length > 0 && accountDataList[0] != null) { + const accountData = accountDataList[0] + return { + h: accountData.stateId, + t: accountData.timestamp || Date.now(), + c: this.stateManager?.currentCycleShardData?.cycleNumber || 0 + } + } + return null + } if (this.accountsHashCache3.accountHashMap.has(accountId) === false) { return null } diff --git a/src/state-manager/AccountGlobals.ts b/src/state-manager/AccountGlobals.ts index e8ecf667f..8d3b5fb4e 100644 --- a/src/state-manager/AccountGlobals.ts +++ b/src/state-manager/AccountGlobals.ts @@ -316,13 +316,13 @@ class AccountGlobals { /* prettier-ignore */ nestedCountersInstance.countEvent('sync', `DATASYNC: getGlobalListEarly success:${this.hasknownGlobals}`) } - getGlobalDebugReport(): { + async getGlobalDebugReport(): Promise<{ globalAccountSummary: { id: string; state: string; ts: number }[] globalStateHash: string - } { + }> { const globalAccountSummary = [] for (const globalID in this.globalAccountSet.keys()) { - const accountHash = this.stateManager.accountCache.getAccountHash(globalID) + const accountHash = await this.stateManager.accountCache.getAccountHash(globalID) const summaryObj = { id: globalID, state: accountHash.h, ts: accountHash.t } globalAccountSummary.push(summaryObj) } diff --git a/src/state-manager/AccountPatcher.ts b/src/state-manager/AccountPatcher.ts index f476f8680..7f77a1754 100644 --- a/src/state-manager/AccountPatcher.ts +++ b/src/state-manager/AccountPatcher.ts @@ -544,7 +544,7 @@ class AccountPatcher { continue } // check if we have already repaired this account - const accountHashCache = this.stateManager.accountCache.getAccountHash(accountID) + const accountHashCache = await this.stateManager.accountCache.getAccountHash(accountID) if (accountHashCache != null && accountHashCache.h === hash) { nestedCountersInstance.countEvent( 'accountPatcher', @@ -1607,7 +1607,7 @@ class AccountPatcher { } const trieAccount = this.getAccountTreeInfo(id) - const accountHash = this.stateManager.accountCache.getAccountHash(id) + const accountHash = await this.stateManager.accountCache.getAccountHash(id) const accountHashFull = this.stateManager.accountCache.getAccountDebugObject(id) //this.stateManager.accountCache.accountsHashCache3.accountHashMap.get(id) const accountData = await this.app.getAccountDataByList([id]) res.write(`trieAccount: ${Utils.safeStringify(trieAccount)} \n`) @@ -2353,8 +2353,7 @@ class AccountPatcher { if (coverageEntry == null || coverageEntry.firstChoice == null) { const numActiveNodes = this.stateManager.currentCycleShardData.nodes.length this.statemanager_fatal( - `getNodeForQuery null ${coverageEntry == null} ${ - coverageEntry?.firstChoice == null + `getNodeForQuery null ${coverageEntry == null} ${coverageEntry?.firstChoice == null } numActiveNodes:${numActiveNodes}`, `getNodeForQuery null ${coverageEntry == null} ${coverageEntry?.firstChoice == null}` ) @@ -2872,8 +2871,7 @@ class AccountPatcher { /* prettier-ignore */ nestedCountersInstance.countEvent(`accountPatcher`, `not enough votes ${radix} ${utils.makeShortHash(votesMap.bestHash)} uniqueVotes: ${votesMap.allVotes.size}`, 1) this.statemanager_fatal( 'debug findBadAccounts', - `debug findBadAccounts ${cycle}: ${radix} bestVotes${ - votesMap.bestVotes + `debug findBadAccounts ${cycle}: ${radix} bestVotes${votesMap.bestVotes } < minVotes:${minVotes} uniqueVotes: ${votesMap.allVotes.size} ${utils.stringifyReduce(simpleMap)}` ) } @@ -2941,10 +2939,8 @@ class AccountPatcher { } this.statemanager_fatal( 'debug findBadAccounts', - `debug findBadAccounts ${cycle}: ${ - radixToFix.radix - } isInNonConsensusRange: ${hasNonConsensusRange} isInNonStorageRange: ${hasNonStorageRange} bestVotes ${ - votesMap.bestVotes + `debug findBadAccounts ${cycle}: ${radixToFix.radix + } isInNonConsensusRange: ${hasNonConsensusRange} isInNonStorageRange: ${hasNonStorageRange} bestVotes ${votesMap.bestVotes } minVotes:${minVotes} uniqueVotes: ${votesMap.allVotes.size} ${utils.stringifyReduce(simpleMap)}` ) } @@ -3123,7 +3119,7 @@ class AccountPatcher { // (we are not supposed to test syncing ranges , but maybe that is out of phase?) //only do this check if the account is new. It was skipping potential oos situations. - const accountMemData: AccountHashCache = this.stateManager.accountCache.getAccountHash( + const accountMemData: AccountHashCache = await this.stateManager.accountCache.getAccountHash( potentalGoodAcc.accountID ) if (accountMemData != null && accountMemData.h === potentalGoodAcc.hash) { @@ -3698,8 +3694,7 @@ class AccountPatcher { if (logFlags.debug) { this.mainLogger.debug( - `badAccounts cycle: ${cycle}, ourBadAccounts: ${ - results.badAccounts.length + `badAccounts cycle: ${cycle}, ourBadAccounts: ${results.badAccounts.length }, ourBadAccounts: ${Utils.safeStringify(results.badAccounts)}` ) } @@ -3709,8 +3704,7 @@ class AccountPatcher { accountsTheyNeedToRepair = accountsTheyNeedToRepair.concat(results.extraBadAccounts) } this.mainLogger.debug( - `badAccounts cycle: ${cycle}, accountsTheyNeedToRepair: ${ - accountsTheyNeedToRepair.length + `badAccounts cycle: ${cycle}, accountsTheyNeedToRepair: ${accountsTheyNeedToRepair.length }, accountsTheyNeedToRepair: ${Utils.safeStringify(accountsTheyNeedToRepair)}` ) this.requestOtherNodesToRepair(accountsTheyNeedToRepair) @@ -3771,8 +3765,8 @@ class AccountPatcher { for (let i = 0; i < wrappedDataList.length; i++) { let wrappedData: Shardus.WrappedData = wrappedDataList[i] let nodeWeAsked = repairDataResponse.nodes[i] - if (this.stateManager.accountCache.hasAccount(wrappedData.accountId)) { - const accountMemData: AccountHashCache = this.stateManager.accountCache.getAccountHash(wrappedData.accountId) + if (await this.stateManager.accountCache.hasAccount(wrappedData.accountId)) { + const accountMemData: AccountHashCache = await this.stateManager.accountCache.getAccountHash(wrappedData.accountId) // dont allow an older timestamp to overwrite a newer copy of data we have. // we may need to do more work to make sure this can not cause an un repairable situation if (wrappedData.timestamp < accountMemData.t) { @@ -3782,8 +3776,7 @@ class AccountPatcher { 'checkAndSetAccountData updateTooOld', `checkAndSetAccountData updateTooOld ${cycle}: acc:${utils.stringifyReduce( wrappedData.accountId - )} updateTS:${wrappedData.timestamp} updateHash:${utils.stringifyReduce(wrappedData.stateId)} cacheTS:${ - accountMemData.t + )} updateTS:${wrappedData.timestamp} updateHash:${utils.stringifyReduce(wrappedData.stateId)} cacheTS:${accountMemData.t } cacheHash:${utils.stringifyReduce(accountMemData.h)}` ) filterStats.tooOld++ @@ -3982,7 +3975,7 @@ class AccountPatcher { ) } const appliedFixes = Math.max(0, wrappedDataListFiltered.length - failedHashes.length) - /* prettier-ignore */ nestedCountersInstance.countEvent('accountPatcher', 'writeCombinedAccountDataToBackups', Math.max(0,wrappedDataListFiltered.length - failedHashes.length)) + /* prettier-ignore */ nestedCountersInstance.countEvent('accountPatcher', 'writeCombinedAccountDataToBackups', Math.max(0, wrappedDataListFiltered.length - failedHashes.length)) /* prettier-ignore */ nestedCountersInstance.countEvent('accountPatcher', `p.repair applied c:${cycle} bad:${results.badAccounts.length} received:${wrappedDataList.length} failedH: ${failedHashes.length} filtered:${utils.stringifyReduce(filterStats)} stats:${utils.stringifyReduce(results.stats)} getAccountStats: ${utils.stringifyReduce(getAccountStats)} extraBadKeys:${results.extraBadKeys.length}`, appliedFixes) this.stateManager.cycleDebugNotes.patchedAccounts = appliedFixes //per cycle debug info @@ -4000,8 +3993,7 @@ class AccountPatcher { ) this.statemanager_fatal( 'isInSync = false', - `bad accounts cycle:${cycle} bad:${results.badAccounts.length} received:${wrappedDataList.length} failedH: ${ - failedHashes.length + `bad accounts cycle:${cycle} bad:${results.badAccounts.length} received:${wrappedDataList.length} failedH: ${failedHashes.length } filtered:${utils.stringifyReduce(filterStats)} stats:${utils.stringifyReduce( results.stats )} getAccountStats: ${utils.stringifyReduce(getAccountStats)} details: ${utils.stringifyReduceLimit( @@ -4048,7 +4040,7 @@ class AccountPatcher { } if (combinedAccountStateData.length > 0) { await this.stateManager.storage.addAccountStates(combinedAccountStateData) - /* prettier-ignore */ nestedCountersInstance.countEvent('accountPatcher', `p.repair stateTable c:${cycle} acc:#${updatedAccounts.length} st#:${combinedAccountStateData.length} missed#${combinedAccountStateData.length-updatedAccounts.length}`, combinedAccountStateData.length) + /* prettier-ignore */ nestedCountersInstance.countEvent('accountPatcher', `p.repair stateTable c:${cycle} acc:#${updatedAccounts.length} st#:${combinedAccountStateData.length} missed#${combinedAccountStateData.length - updatedAccounts.length}`, combinedAccountStateData.length) } if (wrappedDataListFiltered.length > 0) { @@ -4083,7 +4075,7 @@ class AccountPatcher { failHistoryObject.e = this.failEndCycle failHistoryObject.cycles = this.failEndCycle - this.failStartCycle - /* prettier-ignore */ nestedCountersInstance.countEvent(`accountPatcher`, `inSync again. ${Utils.safeStringify(this.syncFailHistory[this.syncFailHistory.length -1])}`) + /* prettier-ignore */ nestedCountersInstance.countEvent(`accountPatcher`, `inSync again. ${Utils.safeStringify(this.syncFailHistory[this.syncFailHistory.length - 1])}`) //this is not really a fatal log so should be removed eventually. is is somewhat usefull context though when debugging. this.statemanager_fatal(`inSync again`, Utils.safeStringify(this.syncFailHistory)) @@ -4587,8 +4579,7 @@ class AccountPatcher { } stream.write( - `node: ${nodesCovered.id} ${nodesCovered.ipPort}\tgraph: ${partitionGraph}\thome: ${ - nodesCovered.hP + `node: ${nodesCovered.id} ${nodesCovered.ipPort}\tgraph: ${partitionGraph}\thome: ${nodesCovered.hP } data:${Utils.safeStringify(nodesCovered)}\n` ) } diff --git a/src/state-manager/PartitionStats.ts b/src/state-manager/PartitionStats.ts index a561e50b3..c3936b60c 100644 --- a/src/state-manager/PartitionStats.ts +++ b/src/state-manager/PartitionStats.ts @@ -315,8 +315,8 @@ class PartitionStats { } //todo , I think this is redundant and removable now. - hasAccountBeenSeenByStats(accountId: string): boolean { - return this.accountCache.hasAccount(accountId) + async hasAccountBeenSeenByStats(accountId: string): Promise { + return await this.accountCache.hasAccount(accountId) } /** @@ -424,7 +424,7 @@ class PartitionStats { * @param accountDataRaw * @param debugMsg */ - statsDataSummaryInit(cycle: number, accountId: string, accountDataRaw: unknown, debugMsg: string): void { + async statsDataSummaryInit(cycle: number, accountId: string, accountDataRaw: unknown, debugMsg: string): Promise { const opCounter = this.statsProcessCounter++ if (this.invasiveDebugInfo) this.mainLogger.debug( @@ -436,7 +436,7 @@ class PartitionStats { const blob: StateManagerTypes.StateManagerTypes.SummaryBlob = this.getSummaryBlob(accountId) blob.counter++ - if (this.accountCache.hasAccount(accountId)) { + if (await this.accountCache.hasAccount(accountId)) { return } const accountInfo = this.app.getTimestampAndHashFromAccount(accountDataRaw) @@ -513,12 +513,12 @@ class PartitionStats { * @param accountDataAfter * @param debugMsg */ - statsDataSummaryUpdate( + async statsDataSummaryUpdate( cycle: number, accountDataBefore: unknown, accountDataAfter: Shardus.WrappedData, debugMsg: string - ): void { + ): Promise { const opCounter = this.statsProcessCounter++ if (this.invasiveDebugInfo) this.mainLogger.debug( @@ -547,8 +547,8 @@ class PartitionStats { const timestamp = accountDataAfter.timestamp // this.app.getAccountTimestamp(accountId) const hash = accountDataAfter.stateId //this.app.getStateId(accountId) - if (this.accountCache.hasAccount(accountId)) { - const accountMemData: AccountHashCache = this.accountCache.getAccountHash(accountId) + if (await this.accountCache.hasAccount(accountId)) { + const accountMemData: AccountHashCache = await this.accountCache.getAccountHash(accountId) if (accountMemData.t > timestamp) { /* prettier-ignore */ if (logFlags.error) this.mainLogger.error(`statsDataSummaryUpdate: good error?: 2: dont update stats with older data skipping update ${utils.makeShortHash(accountId)} ${debugMsg} ${accountMemData.t} > ${timestamp} afterHash:${utils.makeShortHash(accountDataAfter.stateId)}`) return diff --git a/src/state-manager/TransactionQueue.ts b/src/state-manager/TransactionQueue.ts index 4ead0fd57..79ad0371b 100644 --- a/src/state-manager/TransactionQueue.ts +++ b/src/state-manager/TransactionQueue.ts @@ -6556,7 +6556,7 @@ class TransactionQueue { try { //This is a just in time check to make sure our involved accounts //have not changed after our TX timestamp - const accountsValid = this.checkAccountTimestamps(queueEntry) + const accountsValid = await this.checkAccountTimestamps(queueEntry) if (accountsValid === false) { this.updateTxState(queueEntry, 'consensing') queueEntry.preApplyTXResult = { @@ -8394,7 +8394,7 @@ class TransactionQueue { * * @param queueEntry */ - txWillChangeLocalData(queueEntry: QueueEntry): boolean { + async txWillChangeLocalData(queueEntry: QueueEntry): Promise { //if this TX modifies a global then return true since all nodes own all global accounts. if (queueEntry.globalModification) { return true @@ -8420,7 +8420,7 @@ class TransactionQueue { //if(queueEntry.localKeys[key] === true){ if (hasKey) { - const accountHash = this.stateManager.accountCache.getAccountHash(key) + const accountHash = await this.stateManager.accountCache.getAccountHash(key) if (accountHash != null) { // if the timestamp of the TX is newer than any local writeable keys then this tx will change local data if (timestamp > accountHash.t) { @@ -8441,15 +8441,15 @@ class TransactionQueue { * timestamp newer than our transaction timestamp. * If they do have a newer timestamp we must fail the TX and vote for a TX fail receipt. */ - checkAccountTimestamps(queueEntry: QueueEntry): boolean { + async checkAccountTimestamps(queueEntry: QueueEntry): Promise { for (const accountID of Object.keys(queueEntry.involvedReads)) { - const cacheEntry = this.stateManager.accountCache.getAccountHash(accountID) + const cacheEntry = await this.stateManager.accountCache.getAccountHash(accountID) if (cacheEntry != null && cacheEntry.t >= queueEntry.acceptedTx.timestamp) { return false } } for (const accountID of Object.keys(queueEntry.involvedWrites)) { - const cacheEntry = this.stateManager.accountCache.getAccountHash(accountID) + const cacheEntry = await this.stateManager.accountCache.getAccountHash(accountID) if (cacheEntry != null && cacheEntry.t >= queueEntry.acceptedTx.timestamp) { return false } diff --git a/src/state-manager/TransactionRepair.ts b/src/state-manager/TransactionRepair.ts index edaefbe1c..3da3ff97a 100644 --- a/src/state-manager/TransactionRepair.ts +++ b/src/state-manager/TransactionRepair.ts @@ -233,7 +233,7 @@ class TransactionRepair { const shortKey = utils.stringifyReduce(key) const goalHash = voteHashMap.get(key) const data = writtenAccount.data - const hashObj = this.stateManager.accountCache.getAccountHash(key) + const hashObj = await this.stateManager.accountCache.getAccountHash(key) if (goalHash != null && goalHash === data.stateId) { localReadyRepairs.set(key, data) @@ -334,7 +334,7 @@ class TransactionRepair { continue } - const hashObj = this.stateManager.accountCache.getAccountHash(key) + const hashObj = await this.stateManager.accountCache.getAccountHash(key) if (hashObj != null) { // eslint-disable-next-line security/detect-possible-timing-attacks if (hashObj.h === hash) { @@ -548,8 +548,8 @@ class TransactionRepair { } // if our data is already good no need to ask for it again - if (this.stateManager.accountCache.hasAccount(requestObject.accountId)) { - const accountMemData: AccountHashCache = this.stateManager.accountCache.getAccountHash( + if (await this.stateManager.accountCache.hasAccount(requestObject.accountId)) { + const accountMemData: AccountHashCache = await this.stateManager.accountCache.getAccountHash( requestObject.accountId ) if (accountMemData.h === requestObject.accountHash) { @@ -636,7 +636,7 @@ class TransactionRepair { if (repairFix) { //some temp checking. A just in time check to see if we dont need to save this account. - const hashObj = this.stateManager.accountCache.getAccountHash(key) + const hashObj = await this.stateManager.accountCache.getAccountHash(key) if (hashObj != null && hashObj.t > data.timestamp) { /* prettier-ignore */ nestedCountersInstance.countEvent('repair1', 'skip account repair 2, we have a newer copy') continue @@ -751,7 +751,7 @@ class TransactionRepair { let test4 = false let branch4 = -1 if (isGlobal === false) { - const hash = this.stateManager.accountCache.getAccountHash(data.accountId) + const hash = await this.stateManager.accountCache.getAccountHash(data.accountId) // eslint-disable-next-line security/detect-possible-timing-attacks if (hash != null) { @@ -833,7 +833,7 @@ class TransactionRepair { const badHashKeys = [] const noHashKeys = [] for (const key of keysList) { - const hashObj = this.stateManager.accountCache.getAccountHash(key) + const hashObj = await this.stateManager.accountCache.getAccountHash(key) if (hashObj == null) { noHashKeys.push(key) continue diff --git a/src/state-manager/index.ts b/src/state-manager/index.ts index 11c4b6f51..9190d7381 100644 --- a/src/state-manager/index.ts +++ b/src/state-manager/index.ts @@ -1123,8 +1123,8 @@ class StateManager { } //TODO perf remove this when we are satisfied with the situation //Additional testing to cache if we try to overrite with older data - if (this.accountCache.hasAccount(accountId)) { - const accountMemData: AccountHashCache = this.accountCache.getAccountHash(accountId) + if (await this.accountCache.hasAccount(accountId)) { + const accountMemData: AccountHashCache = await this.accountCache.getAccountHash(accountId) if (timestamp < accountMemData.t) { //should update cache anyway (older value may be needed) @@ -1165,7 +1165,7 @@ class StateManager { } if (processStats) { - if (this.accountCache.hasAccount(accountId)) { + if (await this.accountCache.hasAccount(accountId)) { //TODO STATS BUG.. this is what can cause one form of stats bug. //we may have covered this account in the past, then not covered it, and now we cover it again. Stats doesn't know how to repair // this situation. @@ -2640,7 +2640,7 @@ class StateManager { partitionDump.globalAccountIDs = Array.from(this.accountGlobals.globalAccountSet.keys()) partitionDump.globalAccountIDs.sort() - const { globalAccountSummary, globalStateHash } = this.accountGlobals.getGlobalDebugReport() + const { globalAccountSummary, globalStateHash } = await this.accountGlobals.getGlobalDebugReport() partitionDump.globalAccountSummary = globalAccountSummary partitionDump.globalStateHash = globalStateHash } else { diff --git a/test/unit/src/state-manager/AccountCache.test.ts b/test/unit/src/state-manager/AccountCache.test.ts new file mode 100644 index 000000000..2d074d942 --- /dev/null +++ b/test/unit/src/state-manager/AccountCache.test.ts @@ -0,0 +1,442 @@ +// Mock all dependencies before importing +jest.mock('../../../../src/utils/profiler') +jest.mock('../../../../src/logger') +jest.mock('../../../../src/crypto') +jest.mock('../../../../src/utils/nestedCounters', () => ({ + nestedCountersInstance: { + countEvent: jest.fn() + } +})) +jest.mock('../../../../src/network', () => ({ + shardusGetTime: jest.fn(() => Date.now()) +})) +jest.mock('../../../../src/p2p/NodeList', () => ({ + reset: jest.fn() +})) +jest.mock('../../../../src/p2p/CycleChain', () => ({})) +jest.mock('../../../../src/snapshot', () => ({})) +jest.mock('../../../../src/p2p/Active', () => ({})) + +import AccountCache from '../../../../src/state-manager/AccountCache' +import { AccountHashCache, AccountHashCacheHistory } from '../../../../src/state-manager/state-manager-types' + +describe('AccountCache', () => { + let accountCache: AccountCache + let mockStateManager: any + let mockProfiler: any + let mockApp: any + let mockLogger: any + let mockCrypto: any + let mockConfig: any + + beforeEach(() => { + // Mock StateManager + mockStateManager = { + currentCycleShardData: { + cycleNumber: 100 + }, + statemanager_fatal: jest.fn(), + transactionRepair: {}, + transactionQueue: {}, + accountCache: {}, + accountPatcher: {}, + accountGlobals: {}, + accountSync: {}, + partitionObjects: {}, + partitionStats: {}, + archiverDataSourceHelper: {}, + archiverSyncTracker: {}, + cachedAppDataManager: {}, + dataSourceHelper: {}, + nodeSyncTracker: {} + } + + // Mock Profiler + mockProfiler = { + scopedProfileSectionStart: jest.fn(), + scopedProfileSectionEnd: jest.fn() + } + + // Mock App + mockApp = { + getAccountDataByList: jest.fn() + } + + // Mock Logger + mockLogger = { + getLogger: jest.fn().mockReturnValue({ + debug: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + trace: jest.fn() + }) + } + + // Mock Crypto + mockCrypto = { + sign: jest.fn(), + verify: jest.fn(), + hash: jest.fn() + } + + // Default config with bypassAccountCache disabled + mockConfig = { + stateManager: { + bypassAccountCache: false + } + } + + accountCache = new AccountCache( + mockStateManager, + mockProfiler, + mockApp, + mockLogger, + mockCrypto, + mockConfig + ) + }) + + describe('hasAccount', () => { + const testAccountId = 'test-account-123' + + describe('when bypassAccountCache is false (default behavior)', () => { + beforeEach(() => { + mockConfig.stateManager.bypassAccountCache = false + }) + + it('should return true when account exists in cache', async () => { + // Setup cache to have the account + accountCache.accountsHashCache3.accountHashMap.set(testAccountId, { + lastSeenCycle: 99, + lastSeenSortIndex: -1, + queueIndex: { id: -1, idx: -1 }, + accountHashList: [], + lastStaleCycle: -1, + lastUpdateCycle: -1 + }) + + const result = await accountCache.hasAccount(testAccountId) + + expect(result).toBe(true) + expect(mockApp.getAccountDataByList).not.toHaveBeenCalled() + }) + + it('should return false when account does not exist in cache', async () => { + // Cache is empty by default + const result = await accountCache.hasAccount(testAccountId) + + expect(result).toBe(false) + expect(mockApp.getAccountDataByList).not.toHaveBeenCalled() + }) + }) + + describe('when bypassAccountCache is true', () => { + beforeEach(() => { + mockConfig.stateManager.bypassAccountCache = true + }) + + it('should return true when account exists in storage', async () => { + const mockAccountData = { + accountId: testAccountId, + stateId: 'hash123', + timestamp: Date.now(), + data: { balance: 100 } + } + mockApp.getAccountDataByList.mockResolvedValue([mockAccountData]) + + const result = await accountCache.hasAccount(testAccountId) + + expect(result).toBe(true) + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + + it('should return false when account does not exist in storage', async () => { + mockApp.getAccountDataByList.mockResolvedValue([]) + + const result = await accountCache.hasAccount(testAccountId) + + expect(result).toBe(false) + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + + it('should return false when storage returns null account', async () => { + mockApp.getAccountDataByList.mockResolvedValue([null]) + + const result = await accountCache.hasAccount(testAccountId) + + expect(result).toBe(false) + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + + it('should return false when storage returns undefined', async () => { + mockApp.getAccountDataByList.mockResolvedValue(undefined) + + const result = await accountCache.hasAccount(testAccountId) + + expect(result).toBe(false) + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + + it('should handle storage errors gracefully', async () => { + mockApp.getAccountDataByList.mockRejectedValue(new Error('Storage error')) + + await expect(accountCache.hasAccount(testAccountId)).rejects.toThrow('Storage error') + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + }) + }) + + describe('getAccountHash', () => { + const testAccountId = 'test-account-456' + + describe('when bypassAccountCache is false (default behavior)', () => { + beforeEach(() => { + mockConfig.stateManager.bypassAccountCache = false + }) + + it('should return account hash when account exists in cache', async () => { + const mockAccountHash: AccountHashCache = { + h: 'cached-hash-123', + t: 1234567890, + c: 99 + } + + const mockCacheHistory: AccountHashCacheHistory = { + lastSeenCycle: 99, + lastSeenSortIndex: -1, + queueIndex: { id: -1, idx: -1 }, + accountHashList: [mockAccountHash], + lastStaleCycle: -1, + lastUpdateCycle: -1 + } + + accountCache.accountsHashCache3.accountHashMap.set(testAccountId, mockCacheHistory) + + const result = await accountCache.getAccountHash(testAccountId) + + expect(result).toEqual(mockAccountHash) + expect(mockApp.getAccountDataByList).not.toHaveBeenCalled() + }) + + it('should return null when account does not exist in cache', async () => { + const result = await accountCache.getAccountHash(testAccountId) + + expect(result).toBeNull() + expect(mockApp.getAccountDataByList).not.toHaveBeenCalled() + }) + + it('should return undefined when account exists but has no hash list', async () => { + const mockCacheHistory: AccountHashCacheHistory = { + lastSeenCycle: 99, + lastSeenSortIndex: -1, + queueIndex: { id: -1, idx: -1 }, + accountHashList: [], + lastStaleCycle: -1, + lastUpdateCycle: -1 + } + + accountCache.accountsHashCache3.accountHashMap.set(testAccountId, mockCacheHistory) + + const result = await accountCache.getAccountHash(testAccountId) + + expect(result).toBeUndefined() + expect(mockApp.getAccountDataByList).not.toHaveBeenCalled() + }) + }) + + describe('when bypassAccountCache is true', () => { + beforeEach(() => { + mockConfig.stateManager.bypassAccountCache = true + }) + + it('should return account hash from storage when account exists', async () => { + const mockTimestamp = 1640995200000 // 2022-01-01 + const mockAccountData = { + accountId: testAccountId, + stateId: 'storage-hash-456', + timestamp: mockTimestamp, + data: { balance: 200 } + } + mockApp.getAccountDataByList.mockResolvedValue([mockAccountData]) + + const result = await accountCache.getAccountHash(testAccountId) + + expect(result).toEqual({ + h: 'storage-hash-456', + t: mockTimestamp, + c: 100 // Current cycle number from mockStateManager + }) + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + + it('should use current timestamp when account data has no timestamp', async () => { + const mockAccountData = { + accountId: testAccountId, + stateId: 'storage-hash-789', + data: { balance: 300 } + } + mockApp.getAccountDataByList.mockResolvedValue([mockAccountData]) + + const beforeCall = Date.now() + const result = await accountCache.getAccountHash(testAccountId) + const afterCall = Date.now() + + expect(result!.h).toBe('storage-hash-789') + expect(result!.t).toBeGreaterThanOrEqual(beforeCall) + expect(result!.t).toBeLessThanOrEqual(afterCall) + expect(result!.c).toBe(100) + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + + it('should use cycle 0 when currentCycleShardData is null', async () => { + mockStateManager.currentCycleShardData = null + + const mockAccountData = { + accountId: testAccountId, + stateId: 'storage-hash-999', + timestamp: 1640995200000, + data: { balance: 400 } + } + mockApp.getAccountDataByList.mockResolvedValue([mockAccountData]) + + const result = await accountCache.getAccountHash(testAccountId) + + expect(result).toEqual({ + h: 'storage-hash-999', + t: 1640995200000, + c: 0 + }) + }) + + it('should return null when account does not exist in storage', async () => { + mockApp.getAccountDataByList.mockResolvedValue([]) + + const result = await accountCache.getAccountHash(testAccountId) + + expect(result).toBeNull() + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + + it('should return null when storage returns null account', async () => { + mockApp.getAccountDataByList.mockResolvedValue([null]) + + const result = await accountCache.getAccountHash(testAccountId) + + expect(result).toBeNull() + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + + it('should return null when storage returns undefined', async () => { + mockApp.getAccountDataByList.mockResolvedValue(undefined) + + const result = await accountCache.getAccountHash(testAccountId) + + expect(result).toBeNull() + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + + it('should handle storage errors gracefully', async () => { + mockApp.getAccountDataByList.mockRejectedValue(new Error('Storage error')) + + await expect(accountCache.getAccountHash(testAccountId)).rejects.toThrow('Storage error') + expect(mockApp.getAccountDataByList).toHaveBeenCalledWith([testAccountId]) + }) + }) + }) + + describe('integration tests', () => { + const testAccountId = 'integration-test-account' + + it('should handle switching between bypass modes correctly', async () => { + // Start with cache disabled (bypass enabled) + mockConfig.stateManager.bypassAccountCache = true + + const mockAccountData = { + accountId: testAccountId, + stateId: 'storage-hash-integration', + timestamp: 1640995200000, + data: { balance: 500 } + } + mockApp.getAccountDataByList.mockResolvedValue([mockAccountData]) + + // Test with bypass enabled + let hasAccount = await accountCache.hasAccount(testAccountId) + let accountHash = await accountCache.getAccountHash(testAccountId) + + expect(hasAccount).toBe(true) + expect(accountHash).toEqual({ + h: 'storage-hash-integration', + t: 1640995200000, + c: 100 + }) + expect(mockApp.getAccountDataByList).toHaveBeenCalledTimes(2) + + // Switch to cache enabled (bypass disabled) + mockConfig.stateManager.bypassAccountCache = false + + // Add account to cache + const mockCacheHash: AccountHashCache = { + h: 'cached-hash-integration', + t: 1640995300000, + c: 101 + } + + accountCache.accountsHashCache3.accountHashMap.set(testAccountId, { + lastSeenCycle: 101, + lastSeenSortIndex: -1, + queueIndex: { id: -1, idx: -1 }, + accountHashList: [mockCacheHash], + lastStaleCycle: -1, + lastUpdateCycle: -1 + }) + + // Test with bypass disabled (should use cache) + hasAccount = await accountCache.hasAccount(testAccountId) + accountHash = await accountCache.getAccountHash(testAccountId) + + expect(hasAccount).toBe(true) + expect(accountHash).toEqual(mockCacheHash) + // Should still be 2 calls, not more, since we're using cache now + expect(mockApp.getAccountDataByList).toHaveBeenCalledTimes(2) + }) + + it('should handle concurrent requests correctly', async () => { + mockConfig.stateManager.bypassAccountCache = true + + const mockAccountData = { + accountId: testAccountId, + stateId: 'concurrent-hash', + timestamp: 1640995200000, + data: { balance: 600 } + } + mockApp.getAccountDataByList.mockResolvedValue([mockAccountData]) + + // Make multiple concurrent requests + const promises = [ + accountCache.hasAccount(testAccountId), + accountCache.getAccountHash(testAccountId), + accountCache.hasAccount(testAccountId), + accountCache.getAccountHash(testAccountId) + ] + + const results = await Promise.all(promises) + + expect(results[0]).toBe(true) // hasAccount + expect(results[1]).toEqual({ // getAccountHash + h: 'concurrent-hash', + t: 1640995200000, + c: 100 + }) + expect(results[2]).toBe(true) // hasAccount + expect(results[3]).toEqual({ // getAccountHash + h: 'concurrent-hash', + t: 1640995200000, + c: 100 + }) + + // Should have been called for each request + expect(mockApp.getAccountDataByList).toHaveBeenCalledTimes(4) + }) + }) +}) \ No newline at end of file