diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs index 2dbd2ec70..b3f771589 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs @@ -35,6 +35,7 @@ module Test.Cardano.Db.Mock.Config ( configMetadataEnable, configMetadataDisable, configMetadataKeys, + configPoolStats, mkFingerPrint, mkMutableDir, mkDBSyncEnv, @@ -353,6 +354,10 @@ configMetadataKeys :: SyncNodeConfig -> SyncNodeConfig configMetadataKeys cfg = do cfg {dncInsertOptions = (dncInsertOptions cfg) {sioMetadata = MetadataKeys $ 1 :| []}} +configPoolStats :: SyncNodeConfig -> SyncNodeConfig +configPoolStats cfg = do + cfg {dncInsertOptions = (dncInsertOptions cfg) {sioPoolStats = PoolStatsConfig True}} + initCommandLineArgs :: CommandLineArgs initCommandLineArgs = CommandLineArgs diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs index 96fb45ec8..184ca0163 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs @@ -120,6 +120,9 @@ unitTests iom knownMigrations = , test "rollback stake address cache" Rollback.stakeAddressRollback , test "rollback change order of txs" Rollback.rollbackChangeTxOrder , test "rollback full tx" Rollback.rollbackFullTx + , test "basic pool stats functionality" Rollback.poolStatBasicTest + , test "pool stat rollback no duplicates" Rollback.poolStatRollbackNoDuplicates + , test "pool stat rollback general" Rollback.poolStatRollbackGeneral ] , testGroup "different configs" diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Rollback.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Rollback.hs index 5f1cef9c5..5728bdce8 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Rollback.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Rollback.hs @@ -10,8 +10,12 @@ module Test.Cardano.Db.Mock.Unit.Conway.Rollback ( stakeAddressRollback, rollbackChangeTxOrder, rollbackFullTx, + poolStatBasicTest, + poolStatRollbackNoDuplicates, + poolStatRollbackGeneral, ) where +import qualified Cardano.Db as Db import Cardano.Ledger.Coin (Coin (..)) import Cardano.Ledger.Conway.TxCert (ConwayDelegCert (..), Delegatee (..)) import Cardano.Mock.ChainSync.Server (IOManager (), addBlock, rollback) @@ -25,8 +29,9 @@ import Ouroboros.Network.Block (blockPoint) import Test.Cardano.Db.Mock.Config import Test.Cardano.Db.Mock.Examples (mockBlock0, mockBlock1, mockBlock2) import Test.Cardano.Db.Mock.UnifiedApi -import Test.Cardano.Db.Mock.Validate (assertBlockNoBackoff, assertTxCount) -import Test.Tasty.HUnit (Assertion ()) +import qualified Test.Cardano.Db.Mock.UnifiedApi as Api +import Test.Cardano.Db.Mock.Validate (assertBlockNoBackoff, assertTxCount, runQuery) +import Test.Tasty.HUnit (Assertion (), assertBool, assertEqual) import Prelude (last) simpleRollback :: IOManager -> [(Text, Text)] -> Assertion @@ -291,3 +296,108 @@ rollbackFullTx = assertTxCount dbSync 14 where testLabel = "conwayRollbackFullTx" + +poolStatBasicTest :: IOManager -> [(Text, Text)] -> Assertion +poolStatBasicTest = + withCustomConfigAndDropDB args (Just configPoolStats) conwayConfigDir testLabel $ + \interpreter mockServer dbSync -> do + startDBSync dbSync + + -- Test basic pool stats functionality + void $ Api.registerAllStakeCreds interpreter mockServer + assertBlockNoBackoff dbSync 1 + + -- Create some epochs with pool stats + void $ Api.forgeAndSubmitBlocks interpreter mockServer 200 + assertBlockNoBackoff dbSync 201 + + poolStatCount <- runQuery dbSync Db.queryPoolStatCount + + -- Verify pool stats are created and no duplicates exist + duplicateCount <- runQuery dbSync Db.queryPoolStatDuplicates + assertEqual "Should have no duplicate pool stats" 0 duplicateCount + assertBool "Should have some pool stats" (poolStatCount > 0) + where + args = initCommandLineArgs {claFullMode = False} + testLabel = "conwayPoolStatBasicTest" + +poolStatRollbackNoDuplicates :: IOManager -> [(Text, Text)] -> Assertion +poolStatRollbackNoDuplicates = + withCustomConfigAndDropDB args (Just configPoolStats) conwayConfigDir testLabel $ + \interpreter mockServer dbSync -> do + startDBSync dbSync + + -- Simple setup: create some blocks with pool stats + void $ Api.registerAllStakeCreds interpreter mockServer + void $ Api.forgeAndSubmitBlocks interpreter mockServer 200 -- Fill 2 epochs + assertBlockNoBackoff dbSync 201 + + -- Create rollback point + rollbackBlks <- Api.forgeAndSubmitBlocks interpreter mockServer 50 + assertBlockNoBackoff dbSync 251 + + -- Add more blocks to create additional pool stats + void $ Api.forgeAndSubmitBlocks interpreter mockServer 100 -- Fill 1 more epoch + assertBlockNoBackoff dbSync 351 + + -- Rollback (following exact pattern from bigChain test) + atomically $ rollback mockServer (blockPoint $ last rollbackBlks) + assertBlockNoBackoff dbSync 351 -- Delayed rollback + + -- Re-sync some blocks + void $ Api.forgeAndSubmitBlocks interpreter mockServer 100 + assertBlockNoBackoff dbSync 351 -- Should stay same due to rollback + + -- The main test: no duplicates after rollback + re-sync + duplicateCount <- runQuery dbSync Db.queryPoolStatDuplicates + assertEqual "Should have no duplicate pool stats after rollback" 0 duplicateCount + where + args = initCommandLineArgs {claFullMode = False} + testLabel = "conwayPoolStatRollbackNoDuplicates" + +poolStatRollbackGeneral :: IOManager -> [(Text, Text)] -> Assertion +poolStatRollbackGeneral = + withCustomConfigAndDropDB args (Just configPoolStats) conwayConfigDir testLabel $ + \interpreter mockServer dbSync -> do + startDBSync dbSync + + -- Create pools and stake to generate pool stats + void $ Api.registerAllStakeCreds interpreter mockServer + epochBlks1 <- Api.fillEpochs interpreter mockServer 2 + + -- Create rollback point + rollbackBlks <- Api.forgeAndSubmitBlocks interpreter mockServer 10 + let totalBeforeRollback = length epochBlks1 + length rollbackBlks + 1 + assertBlockNoBackoff dbSync totalBeforeRollback + + -- Check initial pool stat count + initialCount <- runQuery dbSync Db.queryPoolStatCount + + -- Forge more blocks to create additional pool stats + epochBlks2 <- Api.fillEpochs interpreter mockServer 1 + let totalAfterEpoch = totalBeforeRollback + length epochBlks2 + assertBlockNoBackoff dbSync totalAfterEpoch + + -- Verify pool stats increased + afterCount <- runQuery dbSync Db.queryPoolStatCount + assertBool "Pool stats should have increased" (afterCount > initialCount) + + -- Rollback to previous point + atomically $ rollback mockServer (blockPoint $ last rollbackBlks) + assertBlockNoBackoff dbSync totalAfterEpoch -- Delayed rollback + + -- Re-sync the same blocks - should not create duplicates + epochBlks3 <- Api.fillEpochs interpreter mockServer 1 + let finalTotal = totalBeforeRollback + length epochBlks3 + 1 + assertBlockNoBackoff dbSync finalTotal + finalCount <- runQuery dbSync Db.queryPoolStatCount + + -- Verify count matches and no constraint violations occurred + assertEqual "Pool stat count should match after rollback" afterCount finalCount + + -- Also verify no duplicates + duplicateCount <- runQuery dbSync Db.queryPoolStatDuplicates + assertEqual "Should have no duplicate pool stats" 0 duplicateCount + where + args = initCommandLineArgs {claFullMode = False} + testLabel = "conwayPoolStatRollbackGeneral" diff --git a/cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatBasicTest b/cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatBasicTest new file mode 100644 index 000000000..f5dc8425c --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatBasicTest @@ -0,0 +1 @@ +[12,16,18,21,24,30,31,32,33,40,41,42,43,47,52,60,62,70,80,84,86,92,98,100,106,109,110,111,112,127,134,138,146,149,154,166,168,178,183,188,193,194,198,200,202,220,222,223,224,225,231,239,242,247,261,282,283,288,289,301,302,303,308,313,315,316,320,331,334,344,345,363,364,368,369,375,377,381,389,394,407,418,422,425,430,437,438,439,440,447,450,453,454,456,458,461,467,492,499,507,516,524,538,541,544,546,550,567,573,576,577,579,580,586,589,595,597,603,605,609,616,618,619,623,624,634,636,643,644,659,664,665,672,678,692,705,711,712,719,726,730,739,740,743,747,749,751,754,759,762,763,765,767,773,777,786,788,789,794,801,806,807,829,830,832,849,851,853,869,871,874,875,878,882,888,893,895,896,898,899,903,906,908,911,912,913,922,930,932,938,941,944,950,960,963,966,968,972,977,985,986] \ No newline at end of file diff --git a/cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatRollbackGeneral b/cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatRollbackGeneral new file mode 100644 index 000000000..2bec212a2 --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatRollbackGeneral @@ -0,0 +1 @@ +[12,16,18,21,24,30,31,32,33,40,41,42,43,47,52,60,62,70,80,84,86,92,98,100,106,109,110,111,112,127,134,138,146,149,154,166,168,178,183,188,193,194,198,200,202,220,222,223,224,225,231,239,242,247,261,282,283,288,289,301,302,303,308,313,315,316,320,331,334,344,345,363,364,368,369,375,377,381,389,394,407,418,422,425,430,437,438,439,440,447,450,453,454,456,458,461,467,492,499,507,516,524,538,541,544,546,550,567,573,576,577,579,580,586,589,595,597,603,605,609,616,618,619,623,624,634,636,643,644,659,664,665,672,678,692,705,711,712,719,726,730,739,740,743,747,749,751,754,759,762,763,765,767,773,777,786,788,789,794,801,806,807,829,830,832,849,851,853,869,871,874,875,878,882,888,893,895,896,898,899,903,906,908,911,912,913,922,930,932,938,941,944,950,960,963,966,968,972,977,985,986,988,990,991,994,997,1002,1034,1035,1036,1039,1041,1051,1059,1061,1063,1068,1076,1081,1082,1091,1101,1102,1104,1106,1123,1126,1138,1141,1143,1144,1149,1162,1167,1172,1177,1181,1184,1185,1188,1190,1203,1205,1214,1215,1221,1233,1234,1245,1249,1250,1251,1261,1265,1266,1267,1272,1281,1283,1289,1294,1300,1304,1307,1308,1310,1314,1315,1319,1325,1334,1350,1353,1359,1362,1370,1371,1373,1375,1381,1399,1404,1415,1416,1419,1420,1425,1426,1435,1436,1437,1441,1444,1448,1458,1461,1462,1467,1470,1479,1486,1489,1494,1496,1497,1513,1519,1528,1529,1538,1549,1551,1553,1555,1567,1580,1583,1595,1601,1603,1613,1614,1616,1625,1637,1638,1639,1640,1643,1653,1654,1655,1658,1667,1672,1674,1683,1692,1700,1706,1709,1712,1714,1715,1717,1726,1733,1750,1756,1758,1759,1771,1778,1781,1783,1789,1797,1800,1801,1816,1823,1828,1831,1840,1847,1852,1858,1872,1873,1874,1876,1893,1894,1895,1900,1905,1912,1918,1923,1930,1940,1944,1949,1952,1953,1958,1965,1966,1968,1972,1974,1975,1994,2005] \ No newline at end of file diff --git a/cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatRollbackNoDuplicates b/cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatRollbackNoDuplicates new file mode 100644 index 000000000..706f2563d --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/conwayPoolStatRollbackNoDuplicates @@ -0,0 +1 @@ +[12,16,18,21,24,30,31,32,33,40,41,42,43,47,52,60,62,70,80,84,86,92,98,100,106,109,110,111,112,127,134,138,146,149,154,166,168,178,183,188,193,194,198,200,202,220,222,223,224,225,231,239,242,247,261,282,283,288,289,301,302,303,308,313,315,316,320,331,334,344,345,363,364,368,369,375,377,381,389,394,407,418,422,425,430,437,438,439,440,447,450,453,454,456,458,461,467,492,499,507,516,524,538,541,544,546,550,567,573,576,577,579,580,586,589,595,597,603,605,609,616,618,619,623,624,634,636,643,644,659,664,665,672,678,692,705,711,712,719,726,730,739,740,743,747,749,751,754,759,762,763,765,767,773,777,786,788,789,794,801,806,807,829,830,832,849,851,853,869,871,874,875,878,882,888,893,895,896,898,899,903,906,908,911,912,913,922,930,932,938,941,944,950,960,963,966,968,972,977,985,986,988,990,991,994,997,1002,1034,1035,1036,1039,1041,1051,1059,1061,1063,1068,1076,1081,1082,1091,1101,1102,1104,1106,1123,1126,1138,1141,1143,1144,1149,1162,1167,1172,1177,1181,1184,1185,1188,1190,1203,1205,1214,1215,1221,1233,1234,1245,1249,1250,1251,1261,1265,1266,1267,1272,1281,1283,1289,1294,1300,1304,1307,1308,1310,1314,1315,1319,1325,1334,1350,1353,1359,1362,1370,1371,1373,1375,1381,1399,1404,1415,1416,1419,1420,1425,1426,1435,1436,1437,1441,1444,1448,1458,1461,1462,1467,1470,1479,1486,1489,1494,1496,1497,1513,1519,1528,1529,1538,1549,1551,1553,1555,1567,1580,1583,1595,1601,1603,1613,1614,1616,1625,1637,1638,1639,1640,1643,1653,1654,1655,1658,1667,1672,1674,1683,1692,1700,1706,1709,1712,1714,1715,1717,1726,1733,1750,1756,1758,1759,1771,1778,1781,1783,1789,1797,1800,1801,1816,1823,1828,1831,1840,1847,1852,1858,1872,1873,1874,1876,1893,1894,1895,1900,1905,1912,1918,1923,1930,1940,1944,1949,1952,1953,1958,1965,1966,1968,1972,1974,1975,1994,2005,2011,2020,2028,2037,2038,2043,2046,2050,2051,2058,2059,2062,2063,2064,2074,2077,2078,2082,2085,2094,2104,2110,2112,2121,2122,2161,2167,2169,2172,2175,2180,2184,2186,2192,2197,2205,2210,2215,2224,2225,2233,2234,2236,2239,2240,2243,2257,2266,2273,2285,2294,2299,2302,2305,2307,2310,2315] \ No newline at end of file diff --git a/cardano-db/src/Cardano/Db/Operations/Query.hs b/cardano-db/src/Cardano/Db/Operations/Query.hs index 5402f7015..0b25856cf 100644 --- a/cardano-db/src/Cardano/Db/Operations/Query.hs +++ b/cardano-db/src/Cardano/Db/Operations/Query.hs @@ -69,6 +69,9 @@ module Cardano.Db.Operations.Query ( queryReservedTicker, queryReservedTickers, queryDelistedPools, + queryPoolStatCount, + queryPoolStatByEpoch, + queryPoolStatDuplicates, queryOffChainPoolFetchError, existsDelistedPool, -- queries used in tools @@ -109,6 +112,7 @@ import Cardano.Db.Schema.BaseSchema import Cardano.Db.Types import Cardano.Ledger.BaseTypes (CertIx (..), TxIx (..)) import Cardano.Ledger.Credential (Ptr (..), SlotNo32 (..)) +import Cardano.Prelude (Int64) import Cardano.Slotting.Slot (SlotNo (..)) import Control.Monad.Extra (join, whenJust) import Control.Monad.IO.Class (MonadIO) @@ -124,6 +128,7 @@ import Database.Esqueleto.Experimental ( Entity (..), PersistEntity, PersistField, + Single (..), SqlBackend, Value (Value, unValue), asc, @@ -144,6 +149,7 @@ import Database.Esqueleto.Experimental ( on, orderBy, persistIdField, + rawSql, select, selectOne, sum_, @@ -922,6 +928,31 @@ queryDelistedPools = do pure $ delistedPool ^. DelistedPoolHashRaw pure $ unValue <$> res +queryPoolStatCount :: MonadIO m => ReaderT SqlBackend m Word64 +queryPoolStatCount = do + res <- select $ do + _ <- from $ table @PoolStat + pure countRows + pure $ maybe 0 unValue (listToMaybe res) + +queryPoolStatByEpoch :: MonadIO m => Word64 -> ReaderT SqlBackend m Word64 +queryPoolStatByEpoch eNo = do + res <- select $ do + poolStat <- from $ table @PoolStat + where_ (poolStat ^. PoolStatEpochNo ==. val (fromIntegral eNo)) + pure countRows + pure $ maybe 0 unValue (listToMaybe res) + +queryPoolStatDuplicates :: MonadIO m => ReaderT SqlBackend m Word64 +queryPoolStatDuplicates = do + res <- + rawSql + "SELECT COUNT(*) FROM (SELECT pool_hash_id, epoch_no, COUNT(*) as cnt FROM pool_stat GROUP BY pool_hash_id, epoch_no HAVING COUNT(*) > 1) AS duplicates" + [] + case res of + [Single c] -> pure $ fromIntegral (c :: Int64) + _otherwise -> pure 0 + -- Returns also the metadata hash queryOffChainPoolFetchError :: MonadIO m => ByteString -> Maybe UTCTime -> ReaderT SqlBackend m [(OffChainPoolFetchError, ByteString)] queryOffChainPoolFetchError hash Nothing = do diff --git a/schema/migration-2-0045-20252107.sql b/schema/migration-2-0045-20252107.sql new file mode 100644 index 000000000..1b5bf4b32 --- /dev/null +++ b/schema/migration-2-0045-20252107.sql @@ -0,0 +1,35 @@ +CREATE FUNCTION migrate() RETURNS void AS $$ +DECLARE + next_version int ; +BEGIN + SELECT stage_two + 1 INTO next_version FROM schema_version ; + IF next_version = 45 THEN + + -- Remove duplicates first + DELETE FROM pool_stat + WHERE id NOT IN ( + SELECT DISTINCT ON (pool_hash_id, epoch_no) id + FROM pool_stat + ORDER BY pool_hash_id, epoch_no, id + ); + + -- Then add constraint if it doesn't exist + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'unique_pool_stat_epoch' + ) THEN + ALTER TABLE "pool_stat" ADD CONSTRAINT "unique_pool_stat_epoch" + UNIQUE ("pool_hash_id", "epoch_no"); + END IF; + END $$; + + UPDATE schema_version SET stage_two = next_version ; + RAISE NOTICE 'DB has been migrated to stage_two version %', next_version ; + END IF ; +END ; +$$ LANGUAGE plpgsql ; + +SELECT migrate() ; +DROP FUNCTION migrate() ;