fix database debug mode and docs

Author: Cmdv

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Revision history for cardano-db-sync
 
+## 13.7.0.0
+
+### Summary
+- Complete migration from Persistent ORM to Hasql for direct PostgreSQL access.
+
+### Performance Improvements
+- 3-4x faster epoch processing: ~30min → ~8min per epoch
+- Improved cache efficiency: Stake address hit rates increased from 57-79% to 89-99%
+- Reduced memory overhead: Eliminated ORM abstraction layer
+
+### Compatibility
+- PostgreSQL schema remains unchanged
+- Existing database instances compatible without migration
+
 ## 13.6.0.5
 - Fix offchain data so it supports files up to 3MB [#1928]
 - Upgrade to PostgreSQL 17

cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Conway/Scenarios.hs

@@ -5,6 +5,8 @@
 
 #if __GLASGOW_HASKELL__ >= 908
 {-# OPTIONS_GHC -Wno-x-partial #-}
+{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
+{-# HLINT ignore "Use zipWith" #-}
 #endif
 
 module Cardano.Mock.Forging.Tx.Conway.Scenarios (

cardano-chain-gen/test/Test/Cardano/Db/Mock/UnifiedApi.hs

@@ -33,7 +33,7 @@ import qualified Cardano.Mock.Forging.Tx.Conway.Scenarios as Conway
 import Cardano.Mock.Forging.Types
 import Cardano.Slotting.Slot (SlotNo (..))
 import Control.Concurrent.Class.MonadSTM.Strict (atomically)
-import Control.Monad (forM, replicateM)
+import Control.Monad (replicateM)
 import Data.Word (Word64)
 import Ouroboros.Consensus.Cardano.Block (
   BabbageEra,
@@ -63,7 +63,7 @@ forgeNextSkipSlotsFindLeaderAndSubmit interpreter mockServer skipSlots txs' = do
 
 forgeAndSubmitBlocks :: Interpreter -> ServerHandle IO CardanoBlock -> Int -> IO [CardanoBlock]
 forgeAndSubmitBlocks interpreter mockServer blocksToCreate =
-  forM [1 .. blocksToCreate] $ \_ -> forgeNextFindLeaderAndSubmit interpreter mockServer []
+  replicateM blocksToCreate (forgeNextFindLeaderAndSubmit interpreter mockServer [])
 
 withAlonzoFindLeaderAndSubmit ::
   Interpreter ->

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Config.hs

@@ -34,7 +34,6 @@ insertConfig = do
           , sioPoolStats = PoolStatsConfig False
           , sioJsonType = JsonTypeDisable
           , sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
-          , sioDbDebug = False
           }
 
   dncInsertOptions cfg @?= expected

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/JsonbInSchema.hs

@@ -1,4 +1,3 @@
-{-# LANGUAGE NumericUnderscores #-}
 {-# LANGUAGE OverloadedStrings #-}
 
 module Test.Cardano.Db.Mock.Unit.Conway.Config.JsonbInSchema (

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs

@@ -104,7 +104,6 @@ insertConfig = do
           , sioPoolStats = PoolStatsConfig False
           , sioJsonType = JsonTypeDisable
           , sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
-          , sioDbDebug = False
           }
 
   dncInsertOptions cfg @?= expected

cardano-db-sync/app/http-get-json-metadata.hs

@@ -1,5 +1,8 @@
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE OverloadedStrings #-}
+{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
+
+{-# HLINT ignore "Avoid restricted alias" #-}
 
 import Cardano.Db (PoolMetaHash (..), PoolUrl (..), VoteMetaHash (..), VoteUrl (..))
 import qualified Cardano.Db as DB

cardano-db-sync/src/Cardano/DbSync.hs

@@ -241,11 +241,11 @@ runSyncNode metricsSetters trce iomgr dbConnSetting runDelayedMigrationFnc syncN
           threadChannels <- liftIO newThreadChannels
           liftIO $
             race_
-              (runDbThread syncEnv metricsSetters threadChannels) -- Main App thread
+              -- We split the main thread into two parts to allow for graceful shutdown of the main App db thread.
+              (runDbThread syncEnv metricsSetters threadChannels)
               ( mapConcurrently_
                   id
-                  [ -- Non-critical threads
-                    runSyncNodeClient metricsSetters syncEnv iomgr trce threadChannels (enpSocketPath syncNodeParams)
+                  [ runSyncNodeClient metricsSetters syncEnv iomgr trce threadChannels (enpSocketPath syncNodeParams)
                   , runFetchOffChainPoolThread syncEnv syncNodeConfigFromFile
                   , runFetchOffChainVoteThread syncEnv syncNodeConfigFromFile
                   , runLedgerStateWriteThread (getTrace syncEnv) (envLedgerEnv syncEnv)

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -50,11 +50,12 @@ module Cardano.DbSync.Api (
 where
 
 import Control.Concurrent.Class.MonadSTM.Strict (
+  StrictTVar,
   newTBQueueIO,
   newTVarIO,
   readTVar,
   readTVarIO,
-  writeTVar, StrictTVar,
+  writeTVar,
  )
 import Control.Monad.Trans.Maybe (MaybeT (..))
 import qualified Data.Strict.Maybe as Strict
@@ -79,7 +80,7 @@ import qualified Ouroboros.Network.Point as Point
 
 import qualified Cardano.Db as DB
 import Cardano.DbSync.Api.Types
-import Cardano.DbSync.Cache.Types (CacheCapacity (..), newEmptyCache, useNoCache, initCacheStatistics)
+import Cardano.DbSync.Cache.Types (CacheCapacity (..), initCacheStatistics, newEmptyCache, useNoCache)
 import Cardano.DbSync.Config.Cardano
 import Cardano.DbSync.Config.Shelley
 import Cardano.DbSync.Config.Types
@@ -445,21 +446,24 @@ mkSyncEnv trce dbEnv syncOptions protoInfo nw nwMagic systemStart syncNodeConfig
 initEpochStatistics :: MonadIO m => m (StrictTVar IO EpochStatistics)
 initEpochStatistics = do
   curTime <- liftIO getCurrentTime
-  liftIO $ newTVarIO $ EpochStatistics
-    { elsStartTime = curTime
-    , elsCaches = initCacheStatistics
-    , elsUnicodeNull = Map.empty
-    }
+  liftIO $
+    newTVarIO $
+      EpochStatistics
+        { elsStartTime = curTime
+        , elsCaches = initCacheStatistics
+        , elsUnicodeNull = Map.empty
+        }
 
 resetEpochStatistics :: MonadIO m => SyncEnv -> m ()
 resetEpochStatistics syncEnv = liftIO $ do
- curTime <- getCurrentTime
- let newEpochStatsValue = EpochStatistics
-       { elsStartTime = curTime
-       , elsCaches = initCacheStatistics
-       , elsUnicodeNull = Map.empty
-       }
- atomically $ writeTVar (envEpochStatistics syncEnv) newEpochStatsValue
+  curTime <- getCurrentTime
+  let newEpochStatsValue =
+        EpochStatistics
+          { elsStartTime = curTime
+          , elsCaches = initCacheStatistics
+          , elsUnicodeNull = Map.empty
+          }
+  atomically $ writeTVar (envEpochStatistics syncEnv) newEpochStatsValue
 
 -- | 'True' is for in memory points and 'False' for on disk
 getLatestPoints :: SyncEnv -> IO [(CardanoPoint, Bool)]

cardano-db-sync/src/Cardano/DbSync/Api/Types.hs

@@ -1,7 +1,7 @@
 {-# LANGUAGE GADTs #-}
+{-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE NoImplicitPrelude #-}
-{-# LANGUAGE OverloadedStrings #-}
 
 module Cardano.DbSync.Api.Types (
   SyncEnv (..),
@@ -11,24 +11,24 @@ module Cardano.DbSync.Api.Types (
   RunMigration,
   ConsistentLevel (..),
   CurrentEpochNo (..),
-  UnicodeNullSource(..),
+  UnicodeNullSource (..),
   EpochStatistics (..),
   formatUnicodeNullSource,
 ) where
 
 import Control.Concurrent.Class.MonadSTM.Strict (StrictTVar)
 import Control.Concurrent.Class.MonadSTM.Strict.TBQueue (StrictTBQueue)
+import qualified Data.Map.Strict as Map
 import qualified Data.Strict.Maybe as Strict
 import Data.Time.Clock (UTCTime)
 import Ouroboros.Consensus.BlockchainTime.WallClock.Types (SystemStart (..))
 import Ouroboros.Network.Magic (NetworkMagic (..))
-import qualified Data.Map.Strict as Map
 
-import Cardano.Prelude (Bool, Eq, IO, Show, Word64, Ord, Text)
+import Cardano.Prelude (Bool, Eq, IO, Ord, Show, Text, Word64)
 import Cardano.Slotting.Slot (EpochNo (..))
 
 import qualified Cardano.Db as DB
-import Cardano.DbSync.Cache.Types (CacheStatus, CacheStatistics)
+import Cardano.DbSync.Cache.Types (CacheStatistics, CacheStatus)
 import Cardano.DbSync.Config.Types (SyncNodeConfig)
 import Cardano.DbSync.Ledger.Types (HasLedgerEnv)
 import Cardano.DbSync.LocalStateQuery (NoLedgerEnv)
@@ -112,7 +112,6 @@ data UnicodeNullSource
   | PrepareTxMetadata
   deriving (Eq, Ord, Show)
 
-
 formatUnicodeNullSource :: UnicodeNullSource -> Text
 formatUnicodeNullSource source = case source of
   InsertDatum -> "insertDatum: Column 'value' in table 'datum'"