From 67b2e870aa8205c885d298f4c3d28c170cbf7e6f Mon Sep 17 00:00:00 2001 From: Russoul Date: Tue, 16 Sep 2025 20:28:19 +0400 Subject: [PATCH 1/5] cardano-submit-api: Port existing traces and metrics to `trace-dispatcher` --- cardano-submit-api/cardano-submit-api.cabal | 8 + cardano-submit-api/src/Cardano/TxSubmit.hs | 49 +++++- .../src/Cardano/TxSubmit/Metrics.hs | 32 ++-- .../src/Cardano/TxSubmit/Rest/Web.hs | 8 +- .../TxSubmit/Tracing/TraceSubmitApi.hs | 149 ++++++++++++++++++ .../src/Cardano/TxSubmit/Util.hs | 8 +- .../src/Cardano/TxSubmit/Web.hs | 29 ++-- 7 files changed, 258 insertions(+), 25 deletions(-) create mode 100644 cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs diff --git a/cardano-submit-api/cardano-submit-api.cabal b/cardano-submit-api/cardano-submit-api.cabal index 5453d438669..c08a5f380fb 100644 --- a/cardano-submit-api/cardano-submit-api.cabal +++ b/cardano-submit-api/cardano-submit-api.cabal @@ -32,6 +32,8 @@ common project-config -Wunused-packages -fwarn-incomplete-patterns -fwarn-redundant-constraints + -Wno-typed-holes + -fno-defer-type-errors library import: project-config @@ -43,8 +45,11 @@ library , cardano-binary , cardano-cli ^>= 10.12 , cardano-crypto-class ^>= 2.2 + , containers + , ekg-core , http-media , iohk-monitoring + , lens , mtl , network , optparse-applicative-fork @@ -52,12 +57,14 @@ library , ouroboros-network ^>= 0.21.2 , ouroboros-network-protocols , prometheus >= 2.2.4 + , ekg-prometheus-adapter , safe-exceptions , servant , servant-server , streaming-commons , text , transformers-except + , trace-dispatcher , warp , yaml @@ -74,6 +81,7 @@ library , Cardano.TxSubmit.Rest.Types , Cardano.TxSubmit.Rest.Web , Cardano.TxSubmit.Tracing.ToObjectOrphans + , Cardano.TxSubmit.Tracing.TraceSubmitApi , Cardano.TxSubmit.Types , Cardano.TxSubmit.Util , Cardano.TxSubmit.Web diff --git a/cardano-submit-api/src/Cardano/TxSubmit.hs b/cardano-submit-api/src/Cardano/TxSubmit.hs index 15eaf9d9ecd..a75c435b235 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit.hs @@ -10,29 +10,59 @@ module Cardano.TxSubmit import qualified Cardano.BM.Setup as Logging import Cardano.BM.Trace (Trace, logInfo) import qualified Cardano.BM.Trace as Logging +import Cardano.Logging (TraceConfig) +import qualified Cardano.Logging.Configuration as TraceD +import Cardano.Logging.ConfigurationParser (readConfigurationWithDefault) +import qualified Cardano.Logging.Trace as TraceD +import qualified Cardano.Logging.Tracer.Composed as TraceD +import Cardano.Logging.Tracer.EKG (ekgTracer) +import Cardano.Logging.Tracer.Standard (standardTracer) +import Cardano.Logging.Types (BackendConfig (..), + ConfigOption (ConfBackend, ConfSeverity), FormatLogging (HumanFormatColoured), + SeverityF (SeverityF), SeverityS (Info)) +import qualified Cardano.Logging.Types as TraceD import Cardano.TxSubmit.CLI.Parsers (opts) import Cardano.TxSubmit.CLI.Types (ConfigFile (unConfigFile), TxSubmitCommand (..), TxSubmitNodeParams (..)) import Cardano.TxSubmit.Config (GenTxSubmitNodeConfig (..), ToggleLogging (..), TxSubmitNodeConfig, readTxSubmitNodeConfig) import Cardano.TxSubmit.Metrics (registerMetricsServer) +import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) import Cardano.TxSubmit.Web (runTxSubmitServer) import qualified Control.Concurrent.Async as Async +import Control.Lens import Control.Monad.IO.Class (MonadIO (liftIO)) +import Data.Map import Data.Text (Text) +import qualified System.Metrics as EKG +import System.Metrics.Prometheus.Registry (RegistrySample, sample) +import System.Remote.Monitoring.Prometheus (defaultOptions, samplingFrequency, + toPrometheusRegistry) + +defaultTraceConfig :: TraceConfig +defaultTraceConfig = + TraceD.emptyTraceConfig + { TraceD.tcOptions = Data.Map.fromList + [([], [ ConfSeverity (SeverityF (Just Info)) + , ConfBackend [Stdout HumanFormatColoured, EKGBackend]]) + ] + } runTxSubmitWebapi :: TxSubmitNodeParams -> IO () runTxSubmitWebapi tsnp = do tsnc <- readTxSubmitNodeConfig (unConfigFile tspConfigFile) + tracingConfig <- readConfigurationWithDefault (unConfigFile tspConfigFile) defaultTraceConfig trce <- mkTracer tsnc - (metrics, runMetricsServer) <- registerMetricsServer trce tspMetricsPort + (trce', registrySample) <- mkTraceDispatcher tracingConfig + (metrics, runMetricsServer) <- registerMetricsServer trce trce' registrySample tspMetricsPort Async.withAsync - (runTxSubmitServer trce metrics tspWebserverConfig tspProtocol tspNetworkId tspSocketPath) + (runTxSubmitServer trce trce' metrics tspWebserverConfig tspProtocol tspNetworkId tspSocketPath) $ \txSubmitServer -> Async.withAsync runMetricsServer $ \_ -> Async.wait txSubmitServer logInfo trce "runTxSubmitWebapi: Stopping TxSubmit API" + TraceD.traceWith trce' ApplicationStopping where TxSubmitNodeParams { tspProtocol @@ -47,3 +77,18 @@ mkTracer :: TxSubmitNodeConfig -> IO (Trace IO Text) mkTracer enc = case tscToggleLogging enc of LoggingOn -> liftIO $ Logging.setupTrace (Right $ tscLoggingConfig enc) "cardano-tx-submit" LoggingOff -> pure Logging.nullTracer + +mkTraceDispatcher :: TraceConfig -> IO (TraceD.Trace IO TraceSubmitApi, IO RegistrySample) +mkTraceDispatcher config = do + trBase <- standardTracer + ekgStore <- EKG.newStore + -- TODO: (@russoul) trace-dispatcher addes a postfix "counter" instead of "count" is that expected? + -- Also, adding those lines below breaks trace-dispatcher + -- void $ EKG.createCounter "tx_submit_counter" ekgStore + -- void $ EKG.createCounter "tx_submit_failed_counter" ekgStore + let registry = toPrometheusRegistry ekgStore (defaultOptions mempty & samplingFrequency .~ 1) -- Convert EKG metrics store to prometheus metrics registry on-demand + trEkg <- ekgTracer config ekgStore + configReflection <- TraceD.emptyConfigReflection + tr <- TraceD.mkCardanoTracer trBase mempty (Just trEkg) ["TxSubmitApi"] + TraceD.configureTracers configReflection config [tr] + pure (tr, registry >>= sample) diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs b/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs index f799b5e13a3..9cf3018309f 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs @@ -12,37 +12,45 @@ import Cardano.Api.Pretty (textShow) import Cardano.BM.Data.Trace (Trace) import Cardano.BM.Trace (logError, logInfo, logWarning) +import Cardano.Logging.Trace (traceWith) +import qualified Cardano.Logging.Types as TraceD +import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) import Control.Exception.Safe import Control.Monad.Reader (MonadReader (ask), ReaderT (runReaderT)) import Data.Text (Text) import qualified Data.Text as T -import System.Metrics.Prometheus.Concurrent.RegistryT (RegistryT (..), registerGauge, +import System.Metrics.Prometheus.Concurrent.RegistryT (RegistryT (..), registerCounter, runRegistryT, unRegistryT) -import System.Metrics.Prometheus.Http.Scrape (serveMetricsT) -import System.Metrics.Prometheus.Metric.Gauge (Gauge) +import System.Metrics.Prometheus.Http.Scrape (serveMetrics) +import System.Metrics.Prometheus.Metric.Counter +import System.Metrics.Prometheus.Registry (RegistrySample) data TxSubmitMetrics = TxSubmitMetrics - { tsmCount :: Gauge - , tsmFailCount :: Gauge + { tsmCount :: Counter + , tsmFailCount :: Counter } -- | Register metrics server. Returns metrics and an IO action which starts metrics server and should -- be passed to 'withAsync'. registerMetricsServer :: Trace IO Text + -> TraceD.Trace IO TraceSubmitApi + -> IO RegistrySample -> Int -> IO (TxSubmitMetrics, IO ()) -registerMetricsServer tracer metricsPort = +registerMetricsServer tracer tracer' registrySample metricsPort = runRegistryT $ do metrics <- makeMetrics registry <- RegistryT ask let runServer = tryWithPort metricsPort $ \port -> do logInfo tracer $ "Starting metrics server on port " <> textShow port - flip runReaderT registry . unRegistryT $ serveMetricsT port [] + traceWith tracer' $ MetricsServerStarted port + flip runReaderT registry . unRegistryT $ serveMetrics port [] registrySample pure (metrics, runServer) where + -- try opening the metrics server on the specified port, if it fails, try using next. Gives up after 1000 attempts and disables metrics server. tryWithPort :: Int -> (Int -> IO ()) -> IO () tryWithPort startingPort f = go startingPort @@ -50,17 +58,21 @@ registerMetricsServer tracer metricsPort = go port = do catch @_ @IOException (f port) $ \e -> do logWarning tracer $ T.pack $ "Metrics server error: " <> displayException e + traceWith tracer' $ MetricsServerError e if port <= (startingPort + 1000) then do logWarning tracer $ "Could not allocate metrics server port " <> textShow port <> " - trying next available..." + traceWith tracer' $ MetricsServerPortOccupied port go $ port + 1 else do logError tracer $ "Could not allocate any metrics port until " <> textShow port <> " - metrics endpoint disabled" - pure () + traceWith tracer' $ MetricsServerPortNotBound port +-- TODO (@russoul) remove this makeMetrics :: RegistryT IO TxSubmitMetrics makeMetrics = TxSubmitMetrics - <$> registerGauge "tx_submit_count" mempty - <*> registerGauge "tx_submit_fail_count" mempty + <$> registerCounter "tx_submit_legacy_count" mempty + <*> registerCounter "tx_submit_legacy_fail_count" mempty + -- Suffix should be left out in the trace-dispatcher code (asMetrics) diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Rest/Web.hs b/cardano-submit-api/src/Cardano/TxSubmit/Rest/Web.hs index 14a7dc85095..cab1c991f51 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Rest/Web.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Rest/Web.hs @@ -8,6 +8,7 @@ where import Cardano.Api.Pretty (textShow) import Cardano.BM.Trace (Trace, logInfo) +import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi(..)) import Control.Exception (bracket) import Data.Streaming.Network (bindPortTCP) @@ -16,10 +17,12 @@ import Network.Socket (close, getSocketName, withSocketsDo) import Network.Wai.Handler.Warp (Settings, getHost, getPort, runSettingsSocket) import Servant (Application) +import qualified Cardano.Logging.Types as TraceD +import Cardano.Logging.Trace (traceWith) -- | Like 'Network.Wai.Handler.Warp.runSettings', except with better logging. -runSettings :: Trace IO Text -> Settings -> Application -> IO () -runSettings trace settings app = +runSettings :: Trace IO Text -> TraceD.Trace IO TraceSubmitApi -> Settings -> Application -> IO () +runSettings trace trace' settings app = withSocketsDo $ bracket (bindPortTCP (getPort settings) (getHost settings)) @@ -27,5 +30,6 @@ runSettings trace settings app = ( \socket -> do addr <- getSocketName socket logInfo trace $ "Web API listening on port " <> textShow addr + traceWith trace' $ EndpointListeningOnPort addr runSettingsSocket settings socket app ) diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs b/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs new file mode 100644 index 00000000000..c8d4c706cf9 --- /dev/null +++ b/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs @@ -0,0 +1,149 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi(..)) where + +import Cardano.Api (TxId (TxId), TxValidationErrorInCardanoMode (..)) +import Cardano.Api.Pretty (textShow) + +import qualified Cardano.Crypto.Hash.Class as Crypto +import Cardano.Logging.Types +import Cardano.TxSubmit.Types (TxCmdError (..)) + +import Prelude hiding (take) + +import Data.Aeson (Value (String), toJSON) +import Data.Aeson.KeyMap (singleton) +import Data.Aeson.Types ((.=)) +import Data.Text (Text, pack, take) +import Data.Text.Encoding (decodeLatin1) +import GHC.Exception.Type (SomeException, displayException) +import GHC.IO.Exception (IOException) +import Network.Socket (SockAddr) + +data TraceSubmitApi = ApplicationStopping + | EndpointListeningOnPort SockAddr + | EndpointException Text SomeException + | EndpointFailedToSubmitTransaction TxCmdError + | EndpointSubmittedTransaction TxId + | EndpointExiting + | MetricsServerStarted Int + | MetricsServerError IOException + | MetricsServerPortOccupied Int + | MetricsServerPortNotBound Int {- tried ports until that one -} + + +-- | Render the first 16 characters of a transaction ID. +renderMediumTxId :: TxId -> Text +renderMediumTxId (TxId hash) = renderMediumHash hash + +-- | Render the first 16 characters of a hex-encoded hash. +renderMediumHash :: Crypto.Hash crypto a -> Text +renderMediumHash = take 16 . decodeLatin1 . Crypto.hashToBytesAsHex + +renderTxCmdError :: TxCmdError -> Text +renderTxCmdError (TxCmdSocketEnvError socketError) = + "socket env error " <> textShow socketError +renderTxCmdError (TxCmdTxReadError envelopeError) = + "transaction read error " <> textShow envelopeError +renderTxCmdError (TxCmdTxSubmitValidationError e) = + case e of + TxValidationErrorInCardanoMode validationErr -> + "transaction submit error " <> textShow validationErr + TxValidationEraMismatch eraMismatch -> + "transaction submit era mismatch" <> textShow eraMismatch + +instance LogFormatting TraceSubmitApi where + -- TODO (from: @russoul, to: @jutaro) why json object is required instead of, more flexible, arbitrary json value? + forMachine _ ApplicationStopping = mempty + forMachine _ (EndpointListeningOnPort addr) = + singleton "addr" (toJSON (textShow addr)) + forMachine _ (EndpointException txt except) = mconcat + [ + "txt" .= String txt, + "exception" .= String (textShow except) + ] + forMachine _ (EndpointFailedToSubmitTransaction txCmdError) = + singleton "error" (toJSON (renderTxCmdError txCmdError)) + forMachine _ (EndpointSubmittedTransaction txId) = + singleton "txId" (String $ renderMediumTxId txId) + forMachine _ EndpointExiting = mempty + forMachine _ (MetricsServerStarted port) = + singleton "port" (toJSON port) + forMachine _ (MetricsServerError except) = + singleton "exception" (toJSON $ displayException except) + forMachine _ (MetricsServerPortOccupied port) = + singleton "port" (toJSON port) + forMachine _ (MetricsServerPortNotBound port) = + singleton "port" (toJSON port) + + forHuman (MetricsServerStarted port) = + "Starting metrics server on port " <> textShow port + forHuman (EndpointException title except) = + title <> textShow except + forHuman (MetricsServerError txt) = + pack $ "Metrics server error: " <> displayException txt + forHuman (MetricsServerPortOccupied port) = + "Could not allocate metrics server port " <> textShow port <> " - trying next available..." + forHuman (MetricsServerPortNotBound untilPort) = + "Could not allocate any metrics port until " <> textShow untilPort <> " - metrics endpoint disabled" + forHuman ApplicationStopping = + "runTxSubmitWebapi: Stopping TxSubmit API" + forHuman (EndpointListeningOnPort port) = + "Web API listening on port " <> textShow port + forHuman EndpointExiting = + "txSubmitApp: exiting" + forHuman (EndpointFailedToSubmitTransaction err) = + "txSubmitPost: failed to submit transaction: " <> renderTxCmdError err + forHuman (EndpointSubmittedTransaction txId) = + "txSubmitPost: successfully submitted transaction " <> renderMediumTxId txId + + asMetrics (EndpointFailedToSubmitTransaction _) = [CounterM "tx_submit_fail" (Just 1)] + asMetrics (EndpointSubmittedTransaction _) = [CounterM "tx_submit" (Just 1)] + asMetrics _ = [] + +instance MetaTrace TraceSubmitApi where + allNamespaces = [ + Namespace [] ["Application", "Stopping"], + + Namespace [] ["Endpoint", "ListeningOnPort"], + Namespace [] ["Endpoint", "Exception"], + Namespace [] ["Endpoint", "FailedToSubmitTransaction"], + Namespace [] ["Endpoint", "SubmittedTransaction"], + Namespace [] ["Endpoint", "Exiting"], + + Namespace [] ["Metrics", "Started"], + Namespace [] ["Metrics", "Error"], + Namespace [] ["Metrics", "PortOccupied"], + Namespace [] ["Metrics", "PortNotBound"] + + ] + + namespaceFor ApplicationStopping = Namespace [] ["Application", "Stopping"] + namespaceFor (EndpointListeningOnPort _) = Namespace [] ["Endpoint", "ListeningOnPort"] + namespaceFor (EndpointException _ _) = Namespace [] ["Endpoint", "Exception"] + namespaceFor (EndpointFailedToSubmitTransaction _) = Namespace [] ["Endpoint", "FailedToSubmitTransaction"] + namespaceFor (EndpointSubmittedTransaction _) = Namespace [] ["Endpoint", "SubmittedTransaction"] + namespaceFor EndpointExiting = Namespace [] ["Endpoint", "Exiting"] + namespaceFor (MetricsServerStarted _) = Namespace [] ["Metrics", "Started"] + namespaceFor (MetricsServerError _) = Namespace [] ["Metrics", "Error"] + namespaceFor (MetricsServerPortOccupied _) = Namespace [] ["Metrics", "PortOccupied"] + namespaceFor (MetricsServerPortNotBound _) = Namespace [] ["Metrics", "PortNotBound"] + + severityFor (Namespace _ ["Application", "Stopping"]) _ = Just Info + severityFor (Namespace _ ["Endpoint", "ListeningOnPort"]) _ = Just Info + severityFor (Namespace _ ["Endpoint", "Exception"]) _ = Just Error + severityFor (Namespace _ ["Endpoint", "Exiting"]) _ = Just Info + severityFor (Namespace _ ["Endpoint", "FailedToSubmitTransaction"]) _ = Just Info + severityFor (Namespace _ ["Endpoint", "SubmittedTransaction"]) _ = Just Info + severityFor (Namespace _ ["Metrics", "Started"]) _ = Just Info + severityFor (Namespace _ ["Metrics", "Error"]) _ = Just Warning + severityFor (Namespace _ ["Metrics", "PortOccupied"]) _ = Just Warning + severityFor (Namespace _ ["Metrics", "PortNotBound"]) _ = Just Error + severityFor _ _ = Nothing + + -- TODO (@russoul) This seems to be necessary for metrics to work at all, why? + metricsDocFor (Namespace _ ["Endpoint", "FailedToSubmitTransaction"]) = [ ("tx_submit_fail", "Number of failed tx submissions") ] + metricsDocFor (Namespace _ ["Endpoint", "SubmittedTransaction"]) = [ ("tx_submit", "Number of successful tx submissions") ] + metricsDocFor _ = [] + + documentFor _ = Nothing diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Util.hs b/cardano-submit-api/src/Cardano/TxSubmit/Util.hs index 25f1dab7e3e..314bb370297 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Util.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Util.hs @@ -5,6 +5,9 @@ module Cardano.TxSubmit.Util import Cardano.Api (textShow) import Cardano.BM.Trace (Trace, logError) +import Cardano.Logging.Trace (traceWith) +import qualified Cardano.Logging.Types as TraceD +import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) import Prelude @@ -15,12 +18,13 @@ import Data.Text (Text) -- code, the caught exception will not be logged. Therefore wrap all tx submission code that -- is called from network with an exception logger so at least the exception will be -- logged (instead of silently swallowed) and then rethrown. -logException :: Trace IO Text -> Text -> IO a -> IO a -logException tracer txt action = action `catch` logger +logException :: Trace IO Text -> TraceD.Trace IO TraceSubmitApi -> Text -> IO a -> IO a +logException tracer tracer' txt action = action `catch` logger where logger :: SomeException -> IO a logger e = do logError tracer $ txt <> textShow e + traceWith tracer' (EndpointException txt e) throwIO e diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Web.hs b/cardano-submit-api/src/Cardano/TxSubmit/Web.hs index f7ba8f93cf1..53eea49d722 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Web.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Web.hs @@ -23,9 +23,12 @@ import Cardano.Api (AllegraEra, AnyCardanoEra (AnyCardanoEra), AsType import Cardano.Binary (DecoderError (..)) import Cardano.BM.Trace (Trace, logInfo) import qualified Cardano.Crypto.Hash.Class as Crypto +import Cardano.Logging.Trace (traceWith) +import qualified Cardano.Logging.Types as TraceD import Cardano.TxSubmit.Metrics (TxSubmitMetrics (..)) import Cardano.TxSubmit.Rest.Types (WebserverConfig (..), toWarpSettings) import qualified Cardano.TxSubmit.Rest.Web as Web +import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) import Cardano.TxSubmit.Types (EnvSocketError (..), RawCborDecodeError (..), TxCmdError (..), TxSubmitApi, TxSubmitApiRecord (..), TxSubmitWebApiError (TxSubmitFail), renderTxCmdError) @@ -61,7 +64,7 @@ import qualified Data.Text.IO as T import System.Environment (lookupEnv) import qualified System.IO as IO import System.IO (IO) -import qualified System.Metrics.Prometheus.Metric.Gauge as Gauge +import qualified System.Metrics.Prometheus.Metric.Counter as Counter import Text.Show (Show (show)) import qualified Servant @@ -71,30 +74,33 @@ import Servant.Server.Generic (AsServerT) runTxSubmitServer :: Trace IO Text + -> TraceD.Trace IO TraceSubmitApi -> TxSubmitMetrics -> WebserverConfig -> ConsensusModeParams -> NetworkId -> SocketPath -> IO () -runTxSubmitServer trace metrics webserverConfig protocol networkId socketPath = do - logException trace "TxSubmit WebAPI: " $ - Web.runSettings trace (toWarpSettings webserverConfig) $ txSubmitApp trace metrics protocol networkId socketPath +runTxSubmitServer trace trace' metrics webserverConfig protocol networkId socketPath = do + logException trace trace' "TxSubmit WebAPI: " $ + Web.runSettings trace trace' (toWarpSettings webserverConfig) $ txSubmitApp trace trace' metrics protocol networkId socketPath logInfo trace "txSubmitApp: exiting" + traceWith trace' EndpointExiting txSubmitApp :: Trace IO Text + -> TraceD.Trace IO TraceSubmitApi -> TxSubmitMetrics -> ConsensusModeParams -> NetworkId -> SocketPath -> Application -txSubmitApp trace metrics connectInfo networkId socketPath = +txSubmitApp trace trace' metrics connectInfo networkId socketPath = Servant.serve (Proxy :: Proxy TxSubmitApi) (toServant handlers) where handlers :: TxSubmitApiRecord (AsServerT Handler) handlers = TxSubmitApiRecord - { _txSubmitPost = txSubmitPost trace metrics connectInfo networkId socketPath + { _txSubmitPost = txSubmitPost trace trace' metrics connectInfo networkId socketPath } deserialiseOne :: forall b. () @@ -125,13 +131,14 @@ readByteStringTx = firstExceptT TxCmdTxReadError . hoistEither . deserialiseAnyO txSubmitPost :: Trace IO Text + -> TraceD.Trace IO TraceSubmitApi -> TxSubmitMetrics -> ConsensusModeParams -> NetworkId -> SocketPath -> ByteString -> Handler TxId -txSubmitPost trace metrics p@(CardanoModeParams cModeParams) networkId socketPath txBytes = +txSubmitPost trace trace' metrics p@(CardanoModeParams cModeParams) networkId socketPath txBytes = handle $ do InAnyShelleyBasedEra sbe tx <- readByteStringTx txBytes let txInMode = TxInMode sbe tx @@ -166,13 +173,17 @@ txSubmitPost trace metrics p@(CardanoModeParams cModeParams) networkId socketPat liftIO $ logInfo trace $ "txSubmitPost: failed to submit transaction: " <> renderTxCmdError err - liftIO $ Gauge.inc (tsmFailCount metrics) + liftIO $ traceWith trace' $ EndpointFailedToSubmitTransaction err + -- TODO: (@russoul) Next step is to tie the metric action (Gauge.inc) to the trace message + liftIO $ Counter.inc (tsmFailCount metrics) errorResponse (TxSubmitFail err) Right txid -> do liftIO $ logInfo trace $ "txSubmitPost: successfully submitted transaction " <> renderMediumTxId txid - liftIO $ Gauge.inc (tsmCount metrics) + liftIO $ traceWith trace' $ EndpointSubmittedTransaction txid + -- TODO: (@russoul) Next step is to tie the metric action (Gauge.inc) to the trace message + liftIO $ Counter.inc (tsmCount metrics) pure txid -- | Render the first 16 characters of a transaction ID. From aa50d6e0d1458bc33115e6a8250d667aae5345f3 Mon Sep 17 00:00:00 2001 From: Russoul Date: Tue, 23 Sep 2025 18:05:31 +0400 Subject: [PATCH 2/5] cardano-submit-api: Eliminate the legacy tracing system --- cardano-submit-api/cardano-submit-api.cabal | 8 --- cardano-submit-api/src/Cardano/TxSubmit.hs | 36 +++-------- .../src/Cardano/TxSubmit/Config.hs | 63 ------------------- .../src/Cardano/TxSubmit/Metrics.hs | 62 ++++-------------- .../src/Cardano/TxSubmit/Rest/Web.hs | 18 ++---- .../TxSubmit/Tracing/ToObjectOrphans.hs | 42 ------------- .../TxSubmit/Tracing/TraceSubmitApi.hs | 16 ++++- .../src/Cardano/TxSubmit/Util.hs | 10 +-- .../src/Cardano/TxSubmit/Web.hs | 51 ++++----------- 9 files changed, 55 insertions(+), 251 deletions(-) delete mode 100644 cardano-submit-api/src/Cardano/TxSubmit/Config.hs delete mode 100644 cardano-submit-api/src/Cardano/TxSubmit/Tracing/ToObjectOrphans.hs diff --git a/cardano-submit-api/cardano-submit-api.cabal b/cardano-submit-api/cardano-submit-api.cabal index c08a5f380fb..cfe841ce9c7 100644 --- a/cardano-submit-api/cardano-submit-api.cabal +++ b/cardano-submit-api/cardano-submit-api.cabal @@ -32,8 +32,6 @@ common project-config -Wunused-packages -fwarn-incomplete-patterns -fwarn-redundant-constraints - -Wno-typed-holes - -fno-defer-type-errors library import: project-config @@ -48,13 +46,10 @@ library , containers , ekg-core , http-media - , iohk-monitoring - , lens , mtl , network , optparse-applicative-fork , ouroboros-consensus-cardano - , ouroboros-network ^>= 0.21.2 , ouroboros-network-protocols , prometheus >= 2.2.4 , ekg-prometheus-adapter @@ -66,7 +61,6 @@ library , transformers-except , trace-dispatcher , warp - , yaml hs-source-dirs: src @@ -74,13 +68,11 @@ library other-modules: Cardano.TxSubmit.CLI.Parsers , Cardano.TxSubmit.CLI.Types - , Cardano.TxSubmit.Config , Cardano.TxSubmit.Metrics , Cardano.TxSubmit.Orphans , Cardano.TxSubmit.Rest.Parsers , Cardano.TxSubmit.Rest.Types , Cardano.TxSubmit.Rest.Web - , Cardano.TxSubmit.Tracing.ToObjectOrphans , Cardano.TxSubmit.Tracing.TraceSubmitApi , Cardano.TxSubmit.Types , Cardano.TxSubmit.Util diff --git a/cardano-submit-api/src/Cardano/TxSubmit.hs b/cardano-submit-api/src/Cardano/TxSubmit.hs index a75c435b235..d22465252ea 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit.hs @@ -7,12 +7,10 @@ module Cardano.TxSubmit , TxSubmitCommand(..) ) where -import qualified Cardano.BM.Setup as Logging -import Cardano.BM.Trace (Trace, logInfo) -import qualified Cardano.BM.Trace as Logging import Cardano.Logging (TraceConfig) import qualified Cardano.Logging.Configuration as TraceD import Cardano.Logging.ConfigurationParser (readConfigurationWithDefault) +import Cardano.Logging.Trace (traceWith) import qualified Cardano.Logging.Trace as TraceD import qualified Cardano.Logging.Tracer.Composed as TraceD import Cardano.Logging.Tracer.EKG (ekgTracer) @@ -24,21 +22,15 @@ import qualified Cardano.Logging.Types as TraceD import Cardano.TxSubmit.CLI.Parsers (opts) import Cardano.TxSubmit.CLI.Types (ConfigFile (unConfigFile), TxSubmitCommand (..), TxSubmitNodeParams (..)) -import Cardano.TxSubmit.Config (GenTxSubmitNodeConfig (..), ToggleLogging (..), - TxSubmitNodeConfig, readTxSubmitNodeConfig) import Cardano.TxSubmit.Metrics (registerMetricsServer) import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) import Cardano.TxSubmit.Web (runTxSubmitServer) import qualified Control.Concurrent.Async as Async -import Control.Lens -import Control.Monad.IO.Class (MonadIO (liftIO)) import Data.Map -import Data.Text (Text) import qualified System.Metrics as EKG import System.Metrics.Prometheus.Registry (RegistrySample, sample) -import System.Remote.Monitoring.Prometheus (defaultOptions, samplingFrequency, - toPrometheusRegistry) +import System.Remote.Monitoring.Prometheus (defaultOptions, toPrometheusRegistry) defaultTraceConfig :: TraceConfig defaultTraceConfig = @@ -51,18 +43,14 @@ defaultTraceConfig = runTxSubmitWebapi :: TxSubmitNodeParams -> IO () runTxSubmitWebapi tsnp = do - tsnc <- readTxSubmitNodeConfig (unConfigFile tspConfigFile) tracingConfig <- readConfigurationWithDefault (unConfigFile tspConfigFile) defaultTraceConfig - trce <- mkTracer tsnc - (trce', registrySample) <- mkTraceDispatcher tracingConfig - (metrics, runMetricsServer) <- registerMetricsServer trce trce' registrySample tspMetricsPort + (trce, registrySample) <- mkTraceDispatcher tracingConfig Async.withAsync - (runTxSubmitServer trce trce' metrics tspWebserverConfig tspProtocol tspNetworkId tspSocketPath) + (runTxSubmitServer trce tspWebserverConfig tspProtocol tspNetworkId tspSocketPath) $ \txSubmitServer -> - Async.withAsync runMetricsServer $ \_ -> + Async.withAsync (registerMetricsServer trce registrySample tspMetricsPort) $ \_ -> Async.wait txSubmitServer - logInfo trce "runTxSubmitWebapi: Stopping TxSubmit API" - TraceD.traceWith trce' ApplicationStopping + TraceD.traceWith trce ApplicationStopping where TxSubmitNodeParams { tspProtocol @@ -73,22 +61,14 @@ runTxSubmitWebapi tsnp = do , tspConfigFile } = tsnp -mkTracer :: TxSubmitNodeConfig -> IO (Trace IO Text) -mkTracer enc = case tscToggleLogging enc of - LoggingOn -> liftIO $ Logging.setupTrace (Right $ tscLoggingConfig enc) "cardano-tx-submit" - LoggingOff -> pure Logging.nullTracer - mkTraceDispatcher :: TraceConfig -> IO (TraceD.Trace IO TraceSubmitApi, IO RegistrySample) mkTraceDispatcher config = do trBase <- standardTracer ekgStore <- EKG.newStore - -- TODO: (@russoul) trace-dispatcher addes a postfix "counter" instead of "count" is that expected? - -- Also, adding those lines below breaks trace-dispatcher - -- void $ EKG.createCounter "tx_submit_counter" ekgStore - -- void $ EKG.createCounter "tx_submit_failed_counter" ekgStore - let registry = toPrometheusRegistry ekgStore (defaultOptions mempty & samplingFrequency .~ 1) -- Convert EKG metrics store to prometheus metrics registry on-demand + let registry = toPrometheusRegistry ekgStore (defaultOptions mempty) -- Convert EKG metrics store to prometheus metrics registry on-demand trEkg <- ekgTracer config ekgStore configReflection <- TraceD.emptyConfigReflection tr <- TraceD.mkCardanoTracer trBase mempty (Just trEkg) ["TxSubmitApi"] TraceD.configureTracers configReflection config [tr] + traceWith tr ApplicationInitializeMetrics pure (tr, registry >>= sample) diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Config.hs b/cardano-submit-api/src/Cardano/TxSubmit/Config.hs deleted file mode 100644 index 366c8035493..00000000000 --- a/cardano-submit-api/src/Cardano/TxSubmit/Config.hs +++ /dev/null @@ -1,63 +0,0 @@ -{-# LANGUAGE FlexibleInstances #-} -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE ScopedTypeVariables #-} - -module Cardano.TxSubmit.Config - ( TxSubmitNodeConfig - , GenTxSubmitNodeConfig (..) - , readTxSubmitNodeConfig - , ToggleLogging(..) - , ToggleMetrics(..) - ) where - -import qualified Cardano.BM.Configuration as Logging -import qualified Cardano.BM.Configuration.Model as Logging -import qualified Cardano.BM.Data.Configuration as Logging - -import Control.Exception (IOException, catch) -import Data.Aeson (FromJSON (..), Object, Value (..), (.:)) -import qualified Data.Aeson as Aeson -import Data.Aeson.Types (Parser) -import Data.Bool (bool) -import Data.ByteString (ByteString) -import qualified Data.ByteString.Char8 as B8 -import qualified Data.Yaml as Yaml - -type TxSubmitNodeConfig = GenTxSubmitNodeConfig Logging.Configuration - -data ToggleLogging = LoggingOn | LoggingOff deriving (Eq, Show) -data ToggleMetrics = MetricsOn | MetricsOff deriving (Eq, Show) - -data GenTxSubmitNodeConfig a = GenTxSubmitNodeConfig - { tscLoggingConfig :: !a - , tscToggleLogging :: !ToggleLogging - , tscToggleMetrics :: !ToggleMetrics - } - -readTxSubmitNodeConfig :: FilePath -> IO TxSubmitNodeConfig -readTxSubmitNodeConfig fp = do - res <- Yaml.decodeEither' <$> readLoggingConfig - case res of - Left err -> error $ "readTxSubmitNodeConfig: Error parsing config: " <> Yaml.prettyPrintParseException err - Right icr -> convertLogging icr - where - readLoggingConfig :: IO ByteString - readLoggingConfig = - catch (B8.readFile fp) $ \(_ :: IOException) -> - error $ "Cannot find the logging configuration file at : " <> fp - -convertLogging :: GenTxSubmitNodeConfig Logging.Representation -> IO TxSubmitNodeConfig -convertLogging tsc = do - lc <- Logging.setupFromRepresentation $ tscLoggingConfig tsc - pure $ tsc { tscLoggingConfig = lc } - ---------------------------------------------------------------------------------------------------- - -instance FromJSON (GenTxSubmitNodeConfig Logging.Representation) where - parseJSON = Aeson.withObject "top-level" parseGenTxSubmitNodeConfig - -parseGenTxSubmitNodeConfig :: Object -> Parser (GenTxSubmitNodeConfig Logging.Representation) -parseGenTxSubmitNodeConfig o = GenTxSubmitNodeConfig - <$> parseJSON (Object o) - <*> fmap (bool LoggingOff LoggingOn) (o .: "EnableLogging") - <*> fmap (bool MetricsOff MetricsOn) (o .: "EnableLogMetrics") diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs b/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs index 9cf3018309f..526ff9aa49d 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs @@ -1,54 +1,28 @@ -{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} module Cardano.TxSubmit.Metrics - ( TxSubmitMetrics (..) - , makeMetrics - , registerMetricsServer - ) + (registerMetricsServer) where -import Cardano.Api.Pretty (textShow) - -import Cardano.BM.Data.Trace (Trace) -import Cardano.BM.Trace (logError, logInfo, logWarning) import Cardano.Logging.Trace (traceWith) import qualified Cardano.Logging.Types as TraceD import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) import Control.Exception.Safe -import Control.Monad.Reader (MonadReader (ask), ReaderT (runReaderT)) -import Data.Text (Text) -import qualified Data.Text as T -import System.Metrics.Prometheus.Concurrent.RegistryT (RegistryT (..), registerCounter, - runRegistryT, unRegistryT) import System.Metrics.Prometheus.Http.Scrape (serveMetrics) -import System.Metrics.Prometheus.Metric.Counter -import System.Metrics.Prometheus.Registry (RegistrySample) - -data TxSubmitMetrics = TxSubmitMetrics - { tsmCount :: Counter - , tsmFailCount :: Counter - } +import System.Metrics.Prometheus.Registry (RegistrySample) -- | Register metrics server. Returns metrics and an IO action which starts metrics server and should -- be passed to 'withAsync'. registerMetricsServer - :: Trace IO Text - -> TraceD.Trace IO TraceSubmitApi + :: TraceD.Trace IO TraceSubmitApi -> IO RegistrySample -> Int - -> IO (TxSubmitMetrics, IO ()) -registerMetricsServer tracer tracer' registrySample metricsPort = - runRegistryT $ do - metrics <- makeMetrics - registry <- RegistryT ask - let runServer = - tryWithPort metricsPort $ \port -> do - logInfo tracer $ "Starting metrics server on port " <> textShow port - traceWith tracer' $ MetricsServerStarted port - flip runReaderT registry . unRegistryT $ serveMetrics port [] registrySample - pure (metrics, runServer) + -> IO () +registerMetricsServer tracer registrySample metricsPort = do + tryWithPort metricsPort $ \port -> do + traceWith tracer $ MetricsServerStarted port + serveMetrics port [] registrySample where -- try opening the metrics server on the specified port, if it fails, try using next. Gives up after 1000 attempts and disables metrics server. @@ -57,22 +31,10 @@ registerMetricsServer tracer tracer' registrySample metricsPort = where go port = do catch @_ @IOException (f port) $ \e -> do - logWarning tracer $ T.pack $ "Metrics server error: " <> displayException e - traceWith tracer' $ MetricsServerError e + traceWith tracer $ MetricsServerError e if port <= (startingPort + 1000) then do - logWarning tracer $ "Could not allocate metrics server port " <> textShow port <> " - trying next available..." - traceWith tracer' $ MetricsServerPortOccupied port + traceWith tracer $ MetricsServerPortOccupied port go $ port + 1 - else do - logError tracer $ - "Could not allocate any metrics port until " <> textShow port <> " - metrics endpoint disabled" - traceWith tracer' $ MetricsServerPortNotBound port - --- TODO (@russoul) remove this -makeMetrics :: RegistryT IO TxSubmitMetrics -makeMetrics = - TxSubmitMetrics - <$> registerCounter "tx_submit_legacy_count" mempty - <*> registerCounter "tx_submit_legacy_fail_count" mempty - -- Suffix should be left out in the trace-dispatcher code (asMetrics) + else + traceWith tracer $ MetricsServerPortNotBound port diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Rest/Web.hs b/cardano-submit-api/src/Cardano/TxSubmit/Rest/Web.hs index cab1c991f51..e7ff08c3de9 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Rest/Web.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Rest/Web.hs @@ -1,35 +1,29 @@ -{-# LANGUAGE OverloadedStrings #-} - module Cardano.TxSubmit.Rest.Web ( runSettings ) where -import Cardano.Api.Pretty (textShow) -import Cardano.BM.Trace (Trace, logInfo) -import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi(..)) +import Cardano.Logging.Trace (traceWith) +import qualified Cardano.Logging.Types as TraceD +import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) import Control.Exception (bracket) import Data.Streaming.Network (bindPortTCP) -import Data.Text (Text) import Network.Socket (close, getSocketName, withSocketsDo) import Network.Wai.Handler.Warp (Settings, getHost, getPort, runSettingsSocket) import Servant (Application) -import qualified Cardano.Logging.Types as TraceD -import Cardano.Logging.Trace (traceWith) -- | Like 'Network.Wai.Handler.Warp.runSettings', except with better logging. -runSettings :: Trace IO Text -> TraceD.Trace IO TraceSubmitApi -> Settings -> Application -> IO () -runSettings trace trace' settings app = +runSettings :: TraceD.Trace IO TraceSubmitApi -> Settings -> Application -> IO () +runSettings trace settings app = withSocketsDo $ bracket (bindPortTCP (getPort settings) (getHost settings)) close ( \socket -> do addr <- getSocketName socket - logInfo trace $ "Web API listening on port " <> textShow addr - traceWith trace' $ EndpointListeningOnPort addr + traceWith trace $ EndpointListeningOnPort addr runSettingsSocket settings socket app ) diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Tracing/ToObjectOrphans.hs b/cardano-submit-api/src/Cardano/TxSubmit/Tracing/ToObjectOrphans.hs deleted file mode 100644 index 506825f80c8..00000000000 --- a/cardano-submit-api/src/Cardano/TxSubmit/Tracing/ToObjectOrphans.hs +++ /dev/null @@ -1,42 +0,0 @@ -{-# LANGUAGE FlexibleInstances #-} -{-# LANGUAGE MultiParamTypeClasses #-} -{-# LANGUAGE OverloadedStrings #-} - -{-# OPTIONS_GHC -fno-warn-orphans #-} - -module Cardano.TxSubmit.Tracing.ToObjectOrphans () where - -import Cardano.BM.Data.Severity (Severity (Debug, Error, Notice, Warning)) -import Cardano.BM.Data.Tracer (HasPrivacyAnnotation, HasSeverityAnnotation (..), - HasTextFormatter, ToObject (toObject), Transformable (..), trStructured) -import Ouroboros.Network.NodeToClient (ErrorPolicyTrace (..), WithAddr (..)) - -import Data.Aeson ((.=)) -import Data.Text (Text) -import qualified Network.Socket as Socket - -instance HasPrivacyAnnotation (WithAddr Socket.SockAddr ErrorPolicyTrace) -instance HasSeverityAnnotation (WithAddr Socket.SockAddr ErrorPolicyTrace) where - getSeverityAnnotation (WithAddr _ ev) = case ev of - ErrorPolicySuspendPeer {} -> Warning -- peer misbehaved - ErrorPolicySuspendConsumer {} -> Notice -- peer temporarily not useful - ErrorPolicyLocalNodeError {} -> Error - ErrorPolicyResumePeer {} -> Debug - ErrorPolicyKeepSuspended {} -> Debug - ErrorPolicyResumeConsumer {} -> Debug - ErrorPolicyResumeProducer {} -> Debug - ErrorPolicyUnhandledApplicationException {} -> Error - ErrorPolicyUnhandledConnectionException {} -> Error - ErrorPolicyAcceptException {} -> Error - -instance HasTextFormatter (WithAddr Socket.SockAddr ErrorPolicyTrace) where - --- transform @ErrorPolicyTrace@ -instance Transformable Text IO (WithAddr Socket.SockAddr ErrorPolicyTrace) where - trTransformer = trStructured - -instance ToObject (WithAddr Socket.SockAddr ErrorPolicyTrace) where - toObject _verb (WithAddr addr ev) = - mconcat [ "kind" .= ("ErrorPolicyTrace" :: String) - , "address" .= show addr - , "event" .= show ev ] diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs b/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs index c8d4c706cf9..5527769a705 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs @@ -21,6 +21,7 @@ import GHC.IO.Exception (IOException) import Network.Socket (SockAddr) data TraceSubmitApi = ApplicationStopping + | ApplicationInitializeMetrics | EndpointListeningOnPort SockAddr | EndpointException Text SomeException | EndpointFailedToSubmitTransaction TxCmdError @@ -75,6 +76,7 @@ instance LogFormatting TraceSubmitApi where singleton "port" (toJSON port) forMachine _ (MetricsServerPortNotBound port) = singleton "port" (toJSON port) + forMachine _ ApplicationInitializeMetrics = mempty forHuman (MetricsServerStarted port) = "Starting metrics server on port " <> textShow port @@ -88,6 +90,7 @@ instance LogFormatting TraceSubmitApi where "Could not allocate any metrics port until " <> textShow untilPort <> " - metrics endpoint disabled" forHuman ApplicationStopping = "runTxSubmitWebapi: Stopping TxSubmit API" + forHuman ApplicationInitializeMetrics = "Metrics initialized" forHuman (EndpointListeningOnPort port) = "Web API listening on port " <> textShow port forHuman EndpointExiting = @@ -97,13 +100,15 @@ instance LogFormatting TraceSubmitApi where forHuman (EndpointSubmittedTransaction txId) = "txSubmitPost: successfully submitted transaction " <> renderMediumTxId txId - asMetrics (EndpointFailedToSubmitTransaction _) = [CounterM "tx_submit_fail" (Just 1)] - asMetrics (EndpointSubmittedTransaction _) = [CounterM "tx_submit" (Just 1)] + asMetrics (EndpointFailedToSubmitTransaction _) = [CounterM "tx_submit_fail" Nothing] + asMetrics (EndpointSubmittedTransaction _) = [CounterM "tx_submit" Nothing] + asMetrics ApplicationInitializeMetrics = [CounterM "tx_submit_fail" (Just 0), CounterM "tx_submit" (Just 0)] asMetrics _ = [] instance MetaTrace TraceSubmitApi where allNamespaces = [ Namespace [] ["Application", "Stopping"], + Namespace [] ["Application", "InitializeMetrics"], Namespace [] ["Endpoint", "ListeningOnPort"], Namespace [] ["Endpoint", "Exception"], @@ -119,6 +124,7 @@ instance MetaTrace TraceSubmitApi where ] namespaceFor ApplicationStopping = Namespace [] ["Application", "Stopping"] + namespaceFor ApplicationInitializeMetrics = Namespace [] ["Application", "InitializeMetrics"] namespaceFor (EndpointListeningOnPort _) = Namespace [] ["Endpoint", "ListeningOnPort"] namespaceFor (EndpointException _ _) = Namespace [] ["Endpoint", "Exception"] namespaceFor (EndpointFailedToSubmitTransaction _) = Namespace [] ["Endpoint", "FailedToSubmitTransaction"] @@ -130,6 +136,7 @@ instance MetaTrace TraceSubmitApi where namespaceFor (MetricsServerPortNotBound _) = Namespace [] ["Metrics", "PortNotBound"] severityFor (Namespace _ ["Application", "Stopping"]) _ = Just Info + severityFor (Namespace _ ["Application", "InitializeMetrics"]) _ = Just Debug severityFor (Namespace _ ["Endpoint", "ListeningOnPort"]) _ = Just Info severityFor (Namespace _ ["Endpoint", "Exception"]) _ = Just Error severityFor (Namespace _ ["Endpoint", "Exiting"]) _ = Just Info @@ -144,6 +151,11 @@ instance MetaTrace TraceSubmitApi where -- TODO (@russoul) This seems to be necessary for metrics to work at all, why? metricsDocFor (Namespace _ ["Endpoint", "FailedToSubmitTransaction"]) = [ ("tx_submit_fail", "Number of failed tx submissions") ] metricsDocFor (Namespace _ ["Endpoint", "SubmittedTransaction"]) = [ ("tx_submit", "Number of successful tx submissions") ] + metricsDocFor (Namespace _ ["Application", "InitializeMetrics"]) = + [ + ("tx_submit_fail", "Initialize and set the number of successful tx submissions to 0"), + ("tx_submit", "Initialize and set the number of successful tx submissions to 0") + ] metricsDocFor _ = [] documentFor _ = Nothing diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Util.hs b/cardano-submit-api/src/Cardano/TxSubmit/Util.hs index 314bb370297..f17e9de49dd 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Util.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Util.hs @@ -2,9 +2,6 @@ module Cardano.TxSubmit.Util ( logException ) where -import Cardano.Api (textShow) - -import Cardano.BM.Trace (Trace, logError) import Cardano.Logging.Trace (traceWith) import qualified Cardano.Logging.Types as TraceD import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) @@ -18,13 +15,12 @@ import Data.Text (Text) -- code, the caught exception will not be logged. Therefore wrap all tx submission code that -- is called from network with an exception logger so at least the exception will be -- logged (instead of silently swallowed) and then rethrown. -logException :: Trace IO Text -> TraceD.Trace IO TraceSubmitApi -> Text -> IO a -> IO a -logException tracer tracer' txt action = action `catch` logger +logException :: TraceD.Trace IO TraceSubmitApi -> Text -> IO a -> IO a +logException tracer txt action = action `catch` logger where logger :: SomeException -> IO a logger e = do - logError tracer $ txt <> textShow e - traceWith tracer' (EndpointException txt e) + traceWith tracer (EndpointException txt e) throwIO e diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Web.hs b/cardano-submit-api/src/Cardano/TxSubmit/Web.hs index 53eea49d722..7628ebae786 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Web.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Web.hs @@ -21,11 +21,9 @@ import Cardano.Api (AllegraEra, AnyCardanoEra (AnyCardanoEra), AsType getTxBody, getTxId, submitTxToNodeLocal) import Cardano.Binary (DecoderError (..)) -import Cardano.BM.Trace (Trace, logInfo) import qualified Cardano.Crypto.Hash.Class as Crypto import Cardano.Logging.Trace (traceWith) import qualified Cardano.Logging.Types as TraceD -import Cardano.TxSubmit.Metrics (TxSubmitMetrics (..)) import Cardano.TxSubmit.Rest.Types (WebserverConfig (..), toWarpSettings) import qualified Cardano.TxSubmit.Rest.Web as Web import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) @@ -73,34 +71,29 @@ import Servant.API.Generic (toServant) import Servant.Server.Generic (AsServerT) runTxSubmitServer - :: Trace IO Text - -> TraceD.Trace IO TraceSubmitApi - -> TxSubmitMetrics + :: TraceD.Trace IO TraceSubmitApi -> WebserverConfig -> ConsensusModeParams -> NetworkId -> SocketPath -> IO () -runTxSubmitServer trace trace' metrics webserverConfig protocol networkId socketPath = do - logException trace trace' "TxSubmit WebAPI: " $ - Web.runSettings trace trace' (toWarpSettings webserverConfig) $ txSubmitApp trace trace' metrics protocol networkId socketPath - logInfo trace "txSubmitApp: exiting" - traceWith trace' EndpointExiting +runTxSubmitServer trace webserverConfig protocol networkId socketPath = do + logException trace "TxSubmit WebAPI: " $ + Web.runSettings trace (toWarpSettings webserverConfig) $ txSubmitApp trace protocol networkId socketPath + traceWith trace EndpointExiting txSubmitApp - :: Trace IO Text - -> TraceD.Trace IO TraceSubmitApi - -> TxSubmitMetrics + :: TraceD.Trace IO TraceSubmitApi -> ConsensusModeParams -> NetworkId -> SocketPath -> Application -txSubmitApp trace trace' metrics connectInfo networkId socketPath = +txSubmitApp trace connectInfo networkId socketPath = Servant.serve (Proxy :: Proxy TxSubmitApi) (toServant handlers) where handlers :: TxSubmitApiRecord (AsServerT Handler) handlers = TxSubmitApiRecord - { _txSubmitPost = txSubmitPost trace trace' metrics connectInfo networkId socketPath + { _txSubmitPost = txSubmitPost trace connectInfo networkId socketPath } deserialiseOne :: forall b. () @@ -130,15 +123,13 @@ readByteStringTx = firstExceptT TxCmdTxReadError . hoistEither . deserialiseAnyO ] txSubmitPost - :: Trace IO Text - -> TraceD.Trace IO TraceSubmitApi - -> TxSubmitMetrics + :: TraceD.Trace IO TraceSubmitApi -> ConsensusModeParams -> NetworkId -> SocketPath -> ByteString -> Handler TxId -txSubmitPost trace trace' metrics p@(CardanoModeParams cModeParams) networkId socketPath txBytes = +txSubmitPost trace p@(CardanoModeParams cModeParams) networkId socketPath txBytes = handle $ do InAnyShelleyBasedEra sbe tx <- readByteStringTx txBytes let txInMode = TxInMode sbe tx @@ -170,26 +161,8 @@ txSubmitPost trace trace' metrics p@(CardanoModeParams cModeParams) networkId so handleSubmitResult res = case res of Left err -> do - liftIO $ logInfo trace $ - "txSubmitPost: failed to submit transaction: " - <> renderTxCmdError err - liftIO $ traceWith trace' $ EndpointFailedToSubmitTransaction err - -- TODO: (@russoul) Next step is to tie the metric action (Gauge.inc) to the trace message - liftIO $ Counter.inc (tsmFailCount metrics) + liftIO $ traceWith trace $ EndpointFailedToSubmitTransaction err errorResponse (TxSubmitFail err) Right txid -> do - liftIO $ logInfo trace $ - "txSubmitPost: successfully submitted transaction " - <> renderMediumTxId txid - liftIO $ traceWith trace' $ EndpointSubmittedTransaction txid - -- TODO: (@russoul) Next step is to tie the metric action (Gauge.inc) to the trace message - liftIO $ Counter.inc (tsmCount metrics) + liftIO $ traceWith trace $ EndpointSubmittedTransaction txid pure txid - --- | Render the first 16 characters of a transaction ID. -renderMediumTxId :: TxId -> Text -renderMediumTxId (TxId hash) = renderMediumHash hash - --- | Render the first 16 characters of a hex-encoded hash. -renderMediumHash :: Crypto.Hash crypto a -> Text -renderMediumHash = T.take 16 . T.decodeLatin1 . Crypto.hashToBytesAsHex From c1add54bd5e35827166e082751e65478356ae7a5 Mon Sep 17 00:00:00 2001 From: Russoul Date: Thu, 9 Oct 2025 13:40:17 +0400 Subject: [PATCH 3/5] cardano-submit-api: Tidy up the imports --- cardano-submit-api/src/Cardano/TxSubmit.hs | 31 +++++++------------ .../src/Cardano/TxSubmit/Metrics.hs | 5 ++- .../TxSubmit/Tracing/TraceSubmitApi.hs | 2 +- .../src/Cardano/TxSubmit/Util.hs | 5 ++- .../src/Cardano/TxSubmit/Web.hs | 9 +++--- 5 files changed, 21 insertions(+), 31 deletions(-) diff --git a/cardano-submit-api/src/Cardano/TxSubmit.hs b/cardano-submit-api/src/Cardano/TxSubmit.hs index d22465252ea..2a12ada0372 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit.hs @@ -7,18 +7,11 @@ module Cardano.TxSubmit , TxSubmitCommand(..) ) where -import Cardano.Logging (TraceConfig) -import qualified Cardano.Logging.Configuration as TraceD -import Cardano.Logging.ConfigurationParser (readConfigurationWithDefault) -import Cardano.Logging.Trace (traceWith) -import qualified Cardano.Logging.Trace as TraceD -import qualified Cardano.Logging.Tracer.Composed as TraceD -import Cardano.Logging.Tracer.EKG (ekgTracer) -import Cardano.Logging.Tracer.Standard (standardTracer) -import Cardano.Logging.Types (BackendConfig (..), - ConfigOption (ConfBackend, ConfSeverity), FormatLogging (HumanFormatColoured), - SeverityF (SeverityF), SeverityS (Info)) -import qualified Cardano.Logging.Types as TraceD +import Cardano.Logging (BackendConfig (..), ConfigOption (ConfBackend, ConfSeverity), + FormatLogging (HumanFormatColoured), SeverityF (SeverityF), SeverityS (Info), + Trace, TraceConfig, configureTracers, ekgTracer, emptyConfigReflection, + emptyTraceConfig, mkCardanoTracer, readConfigurationWithDefault, standardTracer, + tcOptions, traceWith) import Cardano.TxSubmit.CLI.Parsers (opts) import Cardano.TxSubmit.CLI.Types (ConfigFile (unConfigFile), TxSubmitCommand (..), TxSubmitNodeParams (..)) @@ -34,8 +27,8 @@ import System.Remote.Monitoring.Prometheus (defaultOptions, toPromethe defaultTraceConfig :: TraceConfig defaultTraceConfig = - TraceD.emptyTraceConfig - { TraceD.tcOptions = Data.Map.fromList + emptyTraceConfig + { tcOptions = Data.Map.fromList [([], [ ConfSeverity (SeverityF (Just Info)) , ConfBackend [Stdout HumanFormatColoured, EKGBackend]]) ] @@ -50,7 +43,7 @@ runTxSubmitWebapi tsnp = do $ \txSubmitServer -> Async.withAsync (registerMetricsServer trce registrySample tspMetricsPort) $ \_ -> Async.wait txSubmitServer - TraceD.traceWith trce ApplicationStopping + traceWith trce ApplicationStopping where TxSubmitNodeParams { tspProtocol @@ -61,14 +54,14 @@ runTxSubmitWebapi tsnp = do , tspConfigFile } = tsnp -mkTraceDispatcher :: TraceConfig -> IO (TraceD.Trace IO TraceSubmitApi, IO RegistrySample) +mkTraceDispatcher :: TraceConfig -> IO (Trace IO TraceSubmitApi, IO RegistrySample) mkTraceDispatcher config = do trBase <- standardTracer ekgStore <- EKG.newStore let registry = toPrometheusRegistry ekgStore (defaultOptions mempty) -- Convert EKG metrics store to prometheus metrics registry on-demand trEkg <- ekgTracer config ekgStore - configReflection <- TraceD.emptyConfigReflection - tr <- TraceD.mkCardanoTracer trBase mempty (Just trEkg) ["TxSubmitApi"] - TraceD.configureTracers configReflection config [tr] + configReflection <- emptyConfigReflection + tr <- mkCardanoTracer trBase mempty (Just trEkg) ["TxSubmitApi"] + configureTracers configReflection config [tr] traceWith tr ApplicationInitializeMetrics pure (tr, registry >>= sample) diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs b/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs index 526ff9aa49d..c2ab649aa01 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Metrics.hs @@ -4,8 +4,7 @@ module Cardano.TxSubmit.Metrics (registerMetricsServer) where -import Cardano.Logging.Trace (traceWith) -import qualified Cardano.Logging.Types as TraceD +import Cardano.Logging (Trace, traceWith) import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) import Control.Exception.Safe @@ -15,7 +14,7 @@ import System.Metrics.Prometheus.Registry (RegistrySample) -- | Register metrics server. Returns metrics and an IO action which starts metrics server and should -- be passed to 'withAsync'. registerMetricsServer - :: TraceD.Trace IO TraceSubmitApi + :: Trace IO TraceSubmitApi -> IO RegistrySample -> Int -> IO () diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs b/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs index 5527769a705..5a5e2ea2b49 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Tracing/TraceSubmitApi.hs @@ -6,7 +6,7 @@ import Cardano.Api (TxId (TxId), TxValidationErrorInCardanoMode (..)) import Cardano.Api.Pretty (textShow) import qualified Cardano.Crypto.Hash.Class as Crypto -import Cardano.Logging.Types +import Cardano.Logging import Cardano.TxSubmit.Types (TxCmdError (..)) import Prelude hiding (take) diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Util.hs b/cardano-submit-api/src/Cardano/TxSubmit/Util.hs index f17e9de49dd..04372437809 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Util.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Util.hs @@ -2,8 +2,7 @@ module Cardano.TxSubmit.Util ( logException ) where -import Cardano.Logging.Trace (traceWith) -import qualified Cardano.Logging.Types as TraceD +import Cardano.Logging (Trace, traceWith) import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) import Prelude @@ -15,7 +14,7 @@ import Data.Text (Text) -- code, the caught exception will not be logged. Therefore wrap all tx submission code that -- is called from network with an exception logger so at least the exception will be -- logged (instead of silently swallowed) and then rethrown. -logException :: TraceD.Trace IO TraceSubmitApi -> Text -> IO a -> IO a +logException :: Trace IO TraceSubmitApi -> Text -> IO a -> IO a logException tracer txt action = action `catch` logger where logger :: SomeException -> IO a diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Web.hs b/cardano-submit-api/src/Cardano/TxSubmit/Web.hs index 7628ebae786..f646bb0422a 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Web.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Web.hs @@ -22,8 +22,7 @@ import Cardano.Api (AllegraEra, AnyCardanoEra (AnyCardanoEra), AsType import Cardano.Binary (DecoderError (..)) import qualified Cardano.Crypto.Hash.Class as Crypto -import Cardano.Logging.Trace (traceWith) -import qualified Cardano.Logging.Types as TraceD +import Cardano.Logging (Trace, traceWith) import Cardano.TxSubmit.Rest.Types (WebserverConfig (..), toWarpSettings) import qualified Cardano.TxSubmit.Rest.Web as Web import Cardano.TxSubmit.Tracing.TraceSubmitApi (TraceSubmitApi (..)) @@ -71,7 +70,7 @@ import Servant.API.Generic (toServant) import Servant.Server.Generic (AsServerT) runTxSubmitServer - :: TraceD.Trace IO TraceSubmitApi + :: Trace IO TraceSubmitApi -> WebserverConfig -> ConsensusModeParams -> NetworkId @@ -83,7 +82,7 @@ runTxSubmitServer trace webserverConfig protocol networkId socketPath = do traceWith trace EndpointExiting txSubmitApp - :: TraceD.Trace IO TraceSubmitApi + :: Trace IO TraceSubmitApi -> ConsensusModeParams -> NetworkId -> SocketPath @@ -123,7 +122,7 @@ readByteStringTx = firstExceptT TxCmdTxReadError . hoistEither . deserialiseAnyO ] txSubmitPost - :: TraceD.Trace IO TraceSubmitApi + :: Trace IO TraceSubmitApi -> ConsensusModeParams -> NetworkId -> SocketPath From 90739e75c198d43fa306d10936f10c311071a6a4 Mon Sep 17 00:00:00 2001 From: Russoul Date: Fri, 26 Sep 2025 10:26:59 +0400 Subject: [PATCH 4/5] nix: bump iohkNix input and adjust edge node nixos test --- flake.lock | 7 ++++--- flake.nix | 2 +- nix/nixos/tests/cardano-node-edge.nix | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 088771904d5..08280a825e6 100644 --- a/flake.lock +++ b/flake.lock @@ -621,16 +621,17 @@ "sodium": "sodium" }, "locked": { - "lastModified": 1751421193, - "narHash": "sha256-rklXDo12dfukaSqcEyiYbze3ffRtTl2/WAAQCWfkGiw=", + "lastModified": 1758794104, + "narHash": "sha256-ellak8bQWXLbKHzWd7dd0j9vcZttZ3rLEGsCF1cMxQc=", "owner": "input-output-hk", "repo": "iohk-nix", - "rev": "64ca6f4c0c6db283e2ec457c775bce75173fb319", + "rev": "f9fe3eddaa4bc5fc92b5b928d2406a06f8530477", "type": "github" }, "original": { "owner": "input-output-hk", "repo": "iohk-nix", + "rev": "f9fe3eddaa4bc5fc92b5b928d2406a06f8530477", "type": "github" } }, diff --git a/flake.nix b/flake.nix index 922bde27b5d..8265c6ed435 100644 --- a/flake.nix +++ b/flake.nix @@ -53,7 +53,7 @@ incl.url = "github:divnix/incl"; iohkNix = { - url = "github:input-output-hk/iohk-nix"; + url = "github:input-output-hk/iohk-nix/f9fe3eddaa4bc5fc92b5b928d2406a06f8530477"; inputs.nixpkgs.follows = "nixpkgs"; }; diff --git a/nix/nixos/tests/cardano-node-edge.nix b/nix/nixos/tests/cardano-node-edge.nix index a49cabf8e5e..f05df662847 100644 --- a/nix/nixos/tests/cardano-node-edge.nix +++ b/nix/nixos/tests/cardano-node-edge.nix @@ -40,6 +40,7 @@ in { port = 8101; network = environment; socketPath = config.services.cardano-node.socketPath 0; + config = pkgs.cardanoLib.environments.${environment}.submitApiConfig; }; cardano-tracer = { From de5b3d44c5b9f26842871084eb58517319a69dc1 Mon Sep 17 00:00:00 2001 From: Russoul Date: Tue, 23 Sep 2025 18:19:29 +0400 Subject: [PATCH 5/5] cardano-submit-api: Update CHANGELOG and cabal file --- cardano-submit-api/CHANGELOG.md | 6 ++++++ cardano-submit-api/cardano-submit-api.cabal | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cardano-submit-api/CHANGELOG.md b/cardano-submit-api/CHANGELOG.md index b80c679ea8d..2060f060135 100644 --- a/cardano-submit-api/CHANGELOG.md +++ b/cardano-submit-api/CHANGELOG.md @@ -2,6 +2,12 @@ ## vNext +## 10.2 -- Oct 2025 + +* Replace the older tracing & metric system — `iohk-monitoring` with `trace-dispatcher` + * Change prometheus metric type from `gauge` to `counter` + * Use slightly different prometheus suffix for counters: `counter` instead of `count` + ## 10.0 -- Oct 2024 * Bump for Node 10.0 diff --git a/cardano-submit-api/cardano-submit-api.cabal b/cardano-submit-api/cardano-submit-api.cabal index cfe841ce9c7..f55126aa2c7 100644 --- a/cardano-submit-api/cardano-submit-api.cabal +++ b/cardano-submit-api/cardano-submit-api.cabal @@ -1,7 +1,7 @@ cabal-version: 3.0 name: cardano-submit-api -version: 10.1.1 +version: 10.2.0 synopsis: A web server that allows transactions to be POSTed to the cardano chain description: A web server that allows transactions to be POSTed to the cardano chain. homepage: https://github.com/intersectmbo/cardano-node