From 2cd0d5fef9d25d9880650b9a46b57bf71a9ba5e9 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 08:40:27 +0200 Subject: [PATCH 01/38] Experiments with jacoco-filter logic. --- .github/workflows/jacoco_report.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 78c6d87e..a8d5a4bb 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -79,13 +79,40 @@ jobs: with: python-version: '3.12' + + + - name: Checkout jacoco-filter tool + uses: actions/checkout@v4 + with: + repository: atum-service/jacoco-filter + path: jacoco-filter + + - name: Install jacoco-filter dependencies + working-directory: jacoco-filter + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Build jacoco-filter binary + working-directory: jacoco-filter + run: | + pyinstaller --clean jacoco_filter.spec + echo "Built files:" + find dist -type f + + - name: Run jacoco-filter on this project + run: | + ./jacoco-filter/dist/jacoco-filter/jacoco-filter + + + - name: Check coverage thresholds and add reports in PR comments id: jacoco uses: MoranaApps/jacoco-report@v2 with: token: '${{ secrets.GITHUB_TOKEN }}' paths: | - **/target/*${{ env.scalaShort }}*/jacoco/report/jacoco.xml + **/target/*${{ env.scalaShort }}*/jacoco/report/jacoco.filtered.xml exclude-paths: | database/target/** sensitivity: "detail" From 07d08f2a254e91e417762a129e3bd4ae6ef08b4c Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 08:46:40 +0200 Subject: [PATCH 02/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index a8d5a4bb..463cc2f1 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -84,7 +84,7 @@ jobs: - name: Checkout jacoco-filter tool uses: actions/checkout@v4 with: - repository: atum-service/jacoco-filter + repository: MoranaApps/jacoco-filter path: jacoco-filter - name: Install jacoco-filter dependencies From 05e69eaace34fef8e68196296ffe53072f91cbb9 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 08:54:42 +0200 Subject: [PATCH 03/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 2 +- jacoco_filter.toml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 jacoco_filter.toml diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 463cc2f1..674962f6 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -102,7 +102,7 @@ jobs: - name: Run jacoco-filter on this project run: | - ./jacoco-filter/dist/jacoco-filter/jacoco-filter + ./jacoco-filter/dist/jacoco-filter/jacoco-filter --config jacoco_filter.toml diff --git a/jacoco_filter.toml b/jacoco_filter.toml new file mode 100644 index 00000000..df0f4715 --- /dev/null +++ b/jacoco_filter.toml @@ -0,0 +1,7 @@ +inputs = ["**/jacoco.xml"] +exclude_paths = [] +verbose = true + +inline_rules = [ +] + From ea41aa5bc9ac059f273c28b279ef4c3ffe4c3f85 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 13:20:18 +0200 Subject: [PATCH 04/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 674962f6..6f72169c 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -81,28 +81,19 @@ jobs: - - name: Checkout jacoco-filter tool - uses: actions/checkout@v4 - with: - repository: MoranaApps/jacoco-filter - path: jacoco-filter - - - name: Install jacoco-filter dependencies - working-directory: jacoco-filter + - name: Download latest jacoco-filter Linux binary run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + curl -s https://api.github.com/repos/MoranaApps/jacoco-filter/releases/latest \ + | grep "browser_download_url.*jacoco-filter-linux" \ + | cut -d '"' -f 4 \ + | xargs curl -L -o jacoco-filter - - name: Build jacoco-filter binary - working-directory: jacoco-filter - run: | - pyinstaller --clean jacoco_filter.spec - echo "Built files:" - find dist -type f + chmod +x jacoco-filter + ./jacoco-filter --help || echo "✅ Binary is ready" - name: Run jacoco-filter on this project run: | - ./jacoco-filter/dist/jacoco-filter/jacoco-filter --config jacoco_filter.toml + ./jacoco-filter --config jacoco_filter.toml From 5b985afed4ddbca09be66e413ba77799e714f0b3 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 18:42:23 +0200 Subject: [PATCH 05/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 6f72169c..76c5927e 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -80,21 +80,11 @@ jobs: python-version: '3.12' - - - name: Download latest jacoco-filter Linux binary - run: | - curl -s https://api.github.com/repos/MoranaApps/jacoco-filter/releases/latest \ - | grep "browser_download_url.*jacoco-filter-linux" \ - | cut -d '"' -f 4 \ - | xargs curl -L -o jacoco-filter - - chmod +x jacoco-filter - ./jacoco-filter --help || echo "✅ Binary is ready" - - - name: Run jacoco-filter on this project - run: | - ./jacoco-filter --config jacoco_filter.toml - + - name: Run jacoco-filter + uses: MoranaApps/jacoco-filter@feature/add-action-yml-support + with: + config: jacoco_filter.toml + verbose: true - name: Check coverage thresholds and add reports in PR comments From 58b25ebd4f4bb0ce798ef9b408409321369ee823 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 18:47:18 +0200 Subject: [PATCH 06/38] Debug and experiments. --- agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala index 8e9dba60..beddf18f 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala @@ -27,6 +27,8 @@ import za.co.absa.atum.model.types.basic.AtumPartitionsOps */ trait AtumAgent { + + private[this] var contexts: Map[AtumPartitions, AtumContext] = Map.empty val dispatcher: Dispatcher From 22de274369d0a27926f9a6ce28639d592e13f995 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 18:59:37 +0200 Subject: [PATCH 07/38] Debug and experiments. --- jacoco_filter.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacoco_filter.toml b/jacoco_filter.toml index df0f4715..5f3effce 100644 --- a/jacoco_filter.toml +++ b/jacoco_filter.toml @@ -3,5 +3,5 @@ exclude_paths = [] verbose = true inline_rules = [ + "method:za.co.absa.atum.agent.AtumAgent$#dispatcherFromConfig" ] - From ade6e120338ae312668f240ddbd55a68d5efcf6b Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 19:08:58 +0200 Subject: [PATCH 08/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 2 +- jacoco_filter.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 76c5927e..538b99fc 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -83,7 +83,7 @@ jobs: - name: Run jacoco-filter uses: MoranaApps/jacoco-filter@feature/add-action-yml-support with: - config: jacoco_filter.toml + config: ${{ github.repository }}/jacoco_filter.toml verbose: true diff --git a/jacoco_filter.toml b/jacoco_filter.toml index 5f3effce..aa7bc675 100644 --- a/jacoco_filter.toml +++ b/jacoco_filter.toml @@ -1,4 +1,4 @@ -inputs = ["**/jacoco.xml"] +inputs = ["**/*2.13/**/jacoco.xml"] exclude_paths = [] verbose = true From 5e63281712bf3eaca57b018fca9a01dcbb9a0568 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 19:13:17 +0200 Subject: [PATCH 09/38] Debug and experiments. --- jacoco_filter.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacoco_filter.toml b/jacoco_filter.toml index aa7bc675..818848bc 100644 --- a/jacoco_filter.toml +++ b/jacoco_filter.toml @@ -1,4 +1,4 @@ -inputs = ["**/*2.13/**/jacoco.xml"] +inputs = ["**/*2.13*/**/jacoco.xml"] exclude_paths = [] verbose = true From 9c87408261228ff50ceca5463c56365e461f0f98 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 19:21:23 +0200 Subject: [PATCH 10/38] Debug and experiments. --- jacoco_filter.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacoco_filter.toml b/jacoco_filter.toml index 818848bc..7ccaa82a 100644 --- a/jacoco_filter.toml +++ b/jacoco_filter.toml @@ -1,4 +1,4 @@ -inputs = ["**/*2.13*/**/jacoco.xml"] +inputs = ["*2.13*/jacoco.xml"] exclude_paths = [] verbose = true From 037cba2862a86a2bdd25144c8f0cc5697873b6c9 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Tue, 24 Jun 2025 19:27:57 +0200 Subject: [PATCH 11/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 2 +- jacoco_filter.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 538b99fc..21886c53 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -83,7 +83,7 @@ jobs: - name: Run jacoco-filter uses: MoranaApps/jacoco-filter@feature/add-action-yml-support with: - config: ${{ github.repository }}/jacoco_filter.toml + config: ${{ github.workspace }}/jacoco_filter.toml verbose: true diff --git a/jacoco_filter.toml b/jacoco_filter.toml index 7ccaa82a..818848bc 100644 --- a/jacoco_filter.toml +++ b/jacoco_filter.toml @@ -1,4 +1,4 @@ -inputs = ["*2.13*/jacoco.xml"] +inputs = ["**/*2.13*/**/jacoco.xml"] exclude_paths = [] verbose = true From 2716b313044f2a061be56a6df9c06ca5b7f8110b Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Wed, 25 Jun 2025 12:30:35 +0200 Subject: [PATCH 12/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 1 - jacoco_filter.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 21886c53..4863dca0 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -79,7 +79,6 @@ jobs: with: python-version: '3.12' - - name: Run jacoco-filter uses: MoranaApps/jacoco-filter@feature/add-action-yml-support with: diff --git a/jacoco_filter.toml b/jacoco_filter.toml index 818848bc..fb991c07 100644 --- a/jacoco_filter.toml +++ b/jacoco_filter.toml @@ -2,6 +2,6 @@ inputs = ["**/*2.13*/**/jacoco.xml"] exclude_paths = [] verbose = true -inline_rules = [ +rules = [ "method:za.co.absa.atum.agent.AtumAgent$#dispatcherFromConfig" ] From 5f8b22e94ea434403c23d63b5a7e5eb708947648 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Wed, 25 Jun 2025 12:52:36 +0200 Subject: [PATCH 13/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 4863dca0..7ca2bfe5 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -85,6 +85,13 @@ jobs: config: ${{ github.workspace }}/jacoco_filter.toml verbose: true + - name: Upload JaCoCo XMLs (original and filtered) + uses: actions/upload-artifact@v4 + with: + name: jacoco-xmls + path: | + **/jacoco.xml + **/jacoco.filtered.xml - name: Check coverage thresholds and add reports in PR comments id: jacoco From 883e78d83104e8eb3fab8ce4d21b784bbc782496 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 26 Jun 2025 12:45:27 +0200 Subject: [PATCH 14/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 22 +++--- build.sbt | 13 +++- project/JacocoSetup.scala | 117 ++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 12 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 7ca2bfe5..d87c4ada 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -73,17 +73,19 @@ jobs: - name: Build and run tests with test coverage continue-on-error: true id: jacocorun - run: sbt jacoco + run: | + sbt jacoco + sbt filterJacoco - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Run jacoco-filter - uses: MoranaApps/jacoco-filter@feature/add-action-yml-support - with: - config: ${{ github.workspace }}/jacoco_filter.toml - verbose: true +# - uses: actions/setup-python@v5 +# with: +# python-version: '3.12' +# +# - name: Run jacoco-filter +# uses: MoranaApps/jacoco-filter@feature/add-action-yml-support +# with: +# config: ${{ github.workspace }}/jacoco_filter.toml +# verbose: true - name: Upload JaCoCo XMLs (original and filtered) uses: actions/upload-artifact@v4 diff --git a/build.sbt b/build.sbt index 0c95965b..f6b8882b 100644 --- a/build.sbt +++ b/build.sbt @@ -14,11 +14,11 @@ * limitations under the License. */ -import sbt.Keys.name -import sbt.* import Dependencies.* import Dependencies.Versions.spark3 import VersionAxes.* +import sbt.* +import sbt.Keys.* ThisBuild / scalaVersion := Setup.scala213.asString @@ -141,3 +141,12 @@ lazy val reader = (projectMatrix in file("reader")) ) .addScalaCrossBuild(Setup.clientSupportedScalaVersions, Dependencies.readerDependencies) .dependsOn(model) + +/** + * TBD + */ +lazy val filterJacoco = taskKey[Unit]("Run jacoco-filter on JaCoCo XML files") +filterJacoco := JacocoSetup.filterJacocoTask.value + +//lazy val generateFilteredHtmlReport = taskKey[Unit]("Generate HTML reports from filtered JaCoCo XML files") +//generateFilteredHtmlReport := JacocoSetup.generateFilteredHtmlReportTask.value diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index dde78202..6427e4ec 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -15,10 +15,18 @@ import com.github.sbt.jacoco.JacocoKeys.JacocoReportFormats import com.github.sbt.jacoco.report.JacocoReportSettings +import sbt.* +import sbt.Keys.* + +import scala.sys.process.* import za.co.absa.commons.version.Version +import java.nio.file.{Files, StandardCopyOption} + object JacocoSetup { + private val JacocoCLI = config("jacococli").hide + private val jacocoReportCommonSettings: JacocoReportSettings = JacocoReportSettings( formats = Seq(JacocoReportFormats.HTML, JacocoReportFormats.XML) ) @@ -50,4 +58,113 @@ object JacocoSetup { ) } + // Injectable task + def filterJacocoTask: Def.Initialize[Task[Unit]] = Def.task { + val log = streams.value.log + val os = sys.props("os.name").toLowerCase + val isWindows = os.contains("win") + val isMac = os.contains("mac") + val isLinux = os.contains("linux") + + val zipName = if (isWindows) { + "jacoco-filter-Windows.zip" + } else if (isMac) { + "jacoco-filter-macOS.zip" + } else if (isLinux) { + "jacoco-filter-Linux.zip" + } else { + sys.error(s"Unsupported OS: $os") + } + + val downloadUrl = s"https://github.com/MoranaApps/jacoco-filter/releases/latest/download/$zipName" + val targetDir = target.value + val zipPath = targetDir / zipName + val binaryName = zipName.replace(".zip", "") + val binaryPath = targetDir / binaryName / "jacoco-filter" / binaryName.toLowerCase() + + if (!binaryPath.exists()) { + log.info(s"Downloading $zipName...") + + val connection = new URL(downloadUrl).openConnection() + val inputStream = connection.getInputStream + Files.copy(inputStream, zipPath.toPath, StandardCopyOption.REPLACE_EXISTING) + inputStream.close() + + log.info("Unzipping binary...") + IO.unzip(zipPath, targetDir) + + if (!isWindows) { + binaryPath.setExecutable(true) + } + } else { + log.info(s"Binary already exists: $binaryPath") + } + + val tomlPath = baseDirectory.value / "jacoco_filter.toml" + if (!tomlPath.exists()) { + sys.error(s"Config file not found: $tomlPath") + } + + log.info(s"Running jacoco-filter with config: $tomlPath") + + val cmd = if (isWindows) + Seq("cmd", "/c", binaryPath.getAbsolutePath, "--config", tomlPath.getAbsolutePath) + else + Seq(binaryPath.getAbsolutePath, "--config", tomlPath.getAbsolutePath) + + val exitCode = Process(cmd).! + + if (exitCode != 0) + sys.error(s"jacoco-filter failed with exit code $exitCode") + } + + def generateFilteredHtmlReportTask: Def.Initialize[Task[Unit]] = Def.task { + val log = streams.value.log + val baseDir = baseDirectory.value + val filteredXmlFiles = (baseDir ** "jacoco.filtered.xml").get + + val zipName = "jacoco-0.8.13.zip" + val downloadUrl = s"https://github.com/jacoco/jacoco/releases/latest/download/$zipName" + val targetDir = target.value + val zipPath = targetDir / zipName + val jarPath = targetDir / "jacoco-rls" / "lib" / "jacococli.jar" + + if (!jarPath.exists()) { + log.info(s"Downloading $zipName...") + + val connection = new URL(downloadUrl).openConnection() + val inputStream = connection.getInputStream + Files.copy(inputStream, zipPath.toPath, StandardCopyOption.REPLACE_EXISTING) + inputStream.close() + + log.info("Unzipping zip...") + IO.unzip(zipPath, targetDir / "jacoco-rls") + } else { + log.info(s"Jar file already exists: $jarPath") + } + + if (filteredXmlFiles.isEmpty) { + log.warn("No filtered XML files found. Skipping HTML report generation.") + } else { + filteredXmlFiles.foreach { xml => + log.info(s"Generating html report for: ${xml.getAbsolutePath}") + + val classFilesDir = new File(xml.getParentFile.getParentFile.getParentFile, "classes") + val reportDir = new File(xml.getParentFile, "html-filtered") + IO.createDirectory(reportDir) + + val cmd = Seq( + "java", "-jar", jarPath.getAbsolutePath, + "report", xml.getAbsolutePath, + "--classfiles", classFilesDir.getAbsolutePath, + "--html", reportDir.getAbsolutePath + ) + log.info(s"cmd: ${cmd.mkString(" ")}") + val exitCode = Process(cmd, baseDir).! + if (exitCode != 0) sys.error("JaCoCo CLI report generation failed") + + log.info(s"Filtered HTML report generated at: ${reportDir.getAbsolutePath}") + } + } + } } From e2ae55836f7350d3fd67ca0d3bcc2e81c576d40c Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 26 Jun 2025 12:51:59 +0200 Subject: [PATCH 15/38] Debug and experiments. --- .github/workflows/jacoco_report.yml | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index d87c4ada..c2f3e9c4 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -75,25 +75,12 @@ jobs: id: jacocorun run: | sbt jacoco - sbt filterJacoco - -# - uses: actions/setup-python@v5 -# with: -# python-version: '3.12' -# -# - name: Run jacoco-filter -# uses: MoranaApps/jacoco-filter@feature/add-action-yml-support -# with: -# config: ${{ github.workspace }}/jacoco_filter.toml -# verbose: true - - name: Upload JaCoCo XMLs (original and filtered) - uses: actions/upload-artifact@v4 - with: - name: jacoco-xmls - path: | - **/jacoco.xml - **/jacoco.filtered.xml + - name: Filter jacoco xml files + continue-on-error: true + id: jacocofilter + run: | + sbt filterJacoco - name: Check coverage thresholds and add reports in PR comments id: jacoco From 5e78ae8b3f16322f97c58b823a453ea9206f3d01 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 26 Jun 2025 13:32:31 +0200 Subject: [PATCH 16/38] Debug and experiments. --- project/plugins.sbt | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 80117415..c0fb160b 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -29,21 +29,24 @@ addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") // Plugins to build the server module as a jar file addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.2.0") -// sbt-jacoco dependency downloading -lazy val ow2Version = "9.5" -lazy val jacocoVersion = "0.8.11-absa.1" - -def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" -def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" -addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") -addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") +addSbtPlugin("com.github.sbt" % "sbt-jacoco" % "3.5.0") -addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) -addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) -addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) -addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) -addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) -addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) - -addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") +// sbt-jacoco dependency downloading +//lazy val ow2Version = "9.5" +//lazy val jacocoVersion = "0.8.11-absa.1" +// +//def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" +//def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" +// +//addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") +//addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") +// +//addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) +//addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) +//addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) +//addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) +//addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) +//addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) +// +//addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") From fd327ec3dee29fd85cafda68c6d21e14a245c2ca Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 26 Jun 2025 13:38:27 +0200 Subject: [PATCH 17/38] Debug and experiments. --- jacoco_filter.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacoco_filter.toml b/jacoco_filter.toml index fb991c07..d7c3cfb7 100644 --- a/jacoco_filter.toml +++ b/jacoco_filter.toml @@ -3,5 +3,5 @@ exclude_paths = [] verbose = true rules = [ - "method:za.co.absa.atum.agent.AtumAgent$#dispatcherFromConfig" + "method:za.co.absa.atum.agent.AtumAgent#dispatcherFromConfig*" ] From 68693714011dde2d1e3594580191aa26b978ce17 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 26 Jun 2025 13:54:24 +0200 Subject: [PATCH 18/38] Debug and experiments. --- jacoco_filter.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jacoco_filter.toml b/jacoco_filter.toml index d7c3cfb7..920631b9 100644 --- a/jacoco_filter.toml +++ b/jacoco_filter.toml @@ -3,5 +3,6 @@ exclude_paths = [] verbose = true rules = [ - "method:za.co.absa.atum.agent.AtumAgent#dispatcherFromConfig*" + "method:za.co.absa.atum.agent.AtumAgent#dispatcherFromConfig*", + "method:za.co.absa.atum.agent.AtumAgent$#dispatcherFromConfig*" ] From c01f80aaf77b7709525320866f5b833f275836bc Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 26 Jun 2025 13:59:56 +0200 Subject: [PATCH 19/38] Debug and experiments. --- agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala index 4b2de2c0..0ba26467 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala @@ -47,6 +47,7 @@ trait AtumMeasure extends Measure with MeasurementProcessor { } } + final case class UnknownMeasure(measureName: String, measuredColumns: Seq[String], resultValueType: ResultValueType) extends Measure From 58ed86efdc495af90345250f9dfdcece54a74f6a Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 26 Jun 2025 15:11:21 +0200 Subject: [PATCH 20/38] Final changes. --- README.md | 53 ++++++++++++++++++- .../za/co/absa/atum/agent/model/Measure.scala | 1 - build.sbt | 6 +-- jacoco_filter.toml | 7 ++- project/JacocoSetup.scala | 50 ----------------- project/plugins.sbt | 36 ++++++------- 6 files changed, 75 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index cf563701..211ddaf0 100644 --- a/README.md +++ b/README.md @@ -236,10 +236,59 @@ represents all currently supported measurement types (aka measures): sbt jacoco ``` -Code coverage wil be generated on path: +The HTML report of coverage will be generated on the path: + +``` +{project-root}/{module}/target/jvm-{scala_version}/jacoco/report/html/index.html +``` + +The XML report of coverage will be generated on the path: + +``` +{project-root}/{module}/target/jvm-{scala_version}/jacoco/report/jacoco.xml +``` + +### Exclusion During Coverage Report Generation + +To filter out files from the code coverage report, you can define their sequence in the file `{project-root}/project/JacocoSetup.scala`. + +``` + def jacocoProjectExcludes(): Seq[String] = { + Seq( + "**.api.http.*", + "**.config.*", + "za.co.absa.atum.agent.dispatcher.Dispatcher", + "za.co.absa.atum.agent.dispatcher.ConsoleDispatcher", + "za.co.absa.atum.server.Main*", + "za.co.absa.atum.server.Constants*", + "za.co.absa.atum.server.api.database.DoobieImplicits*", + "za.co.absa.atum.server.api.database.TransactorProvider*", + "za.co.absa.atum.model.envelopes.Pagination", + "za.co.absa.atum.model.envelopes.ResponseEnvelope", + "za.co.absa.atum.model.envelopes.StatusResponse", + "za.co.absa.atum.model.envelopes.SuccessResponse" + ) + } ``` -{project-root}/{module}/target/jvm-{scala_version}/jacoco/report/html + +### Exclusion After Coverage Report Generation + +To filter out files, objects (`class`, `object`, `trait`), and methods from the code coverage report after it has been generated, you can define their sequence in the file `{project-root}/jacoco_filter.toml`. + ``` +rules = [ + # filter-out methods from AtumAgent class + "method:za.co.absa.atum.agent.AtumAgent#dispatcherFromConfig*", + + # filter-out methods from AtumAgent object + "method:za.co.absa.atum.agent.AtumAgent$#dispatcherFromConfig*" +] +``` + +> For filtration, a [Jacoco Filter](https://github.com/MoranaApps/jacoco-filter) tool is used. +>
**Hint:** To identify object names, observe values in the jacoco.xml files with non-zero missed instructions. + +> **Important:** In the used version of the tool, the filtration is applied on the XML report, not on the HTML report! ## How to Run in IntelliJ diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala index 0ba26467..4b2de2c0 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala @@ -47,7 +47,6 @@ trait AtumMeasure extends Measure with MeasurementProcessor { } } - final case class UnknownMeasure(measureName: String, measuredColumns: Seq[String], resultValueType: ResultValueType) extends Measure diff --git a/build.sbt b/build.sbt index f6b8882b..5db8e98e 100644 --- a/build.sbt +++ b/build.sbt @@ -143,10 +143,8 @@ lazy val reader = (projectMatrix in file("reader")) .dependsOn(model) /** - * TBD + * Register a Jacoco filter task that runs the Jacoco-filter script on JaCoCo XML files. + * This task expects a configuration file 'jacoco_filter.toml' to be present in the project root directory. */ lazy val filterJacoco = taskKey[Unit]("Run jacoco-filter on JaCoCo XML files") filterJacoco := JacocoSetup.filterJacocoTask.value - -//lazy val generateFilteredHtmlReport = taskKey[Unit]("Generate HTML reports from filtered JaCoCo XML files") -//generateFilteredHtmlReport := JacocoSetup.generateFilteredHtmlReportTask.value diff --git a/jacoco_filter.toml b/jacoco_filter.toml index 920631b9..4198696b 100644 --- a/jacoco_filter.toml +++ b/jacoco_filter.toml @@ -3,6 +3,9 @@ exclude_paths = [] verbose = true rules = [ - "method:za.co.absa.atum.agent.AtumAgent#dispatcherFromConfig*", - "method:za.co.absa.atum.agent.AtumAgent$#dispatcherFromConfig*" + # filter-out method from AtumAgent class + #"method:za.co.absa.atum.agent.AtumAgent#dispatcherFromConfig*", + + # filter-out method from AtumAgent object + #"method:za.co.absa.atum.agent.AtumAgent$#dispatcherFromConfig*" ] diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index 6427e4ec..009e3189 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -117,54 +117,4 @@ object JacocoSetup { if (exitCode != 0) sys.error(s"jacoco-filter failed with exit code $exitCode") } - - def generateFilteredHtmlReportTask: Def.Initialize[Task[Unit]] = Def.task { - val log = streams.value.log - val baseDir = baseDirectory.value - val filteredXmlFiles = (baseDir ** "jacoco.filtered.xml").get - - val zipName = "jacoco-0.8.13.zip" - val downloadUrl = s"https://github.com/jacoco/jacoco/releases/latest/download/$zipName" - val targetDir = target.value - val zipPath = targetDir / zipName - val jarPath = targetDir / "jacoco-rls" / "lib" / "jacococli.jar" - - if (!jarPath.exists()) { - log.info(s"Downloading $zipName...") - - val connection = new URL(downloadUrl).openConnection() - val inputStream = connection.getInputStream - Files.copy(inputStream, zipPath.toPath, StandardCopyOption.REPLACE_EXISTING) - inputStream.close() - - log.info("Unzipping zip...") - IO.unzip(zipPath, targetDir / "jacoco-rls") - } else { - log.info(s"Jar file already exists: $jarPath") - } - - if (filteredXmlFiles.isEmpty) { - log.warn("No filtered XML files found. Skipping HTML report generation.") - } else { - filteredXmlFiles.foreach { xml => - log.info(s"Generating html report for: ${xml.getAbsolutePath}") - - val classFilesDir = new File(xml.getParentFile.getParentFile.getParentFile, "classes") - val reportDir = new File(xml.getParentFile, "html-filtered") - IO.createDirectory(reportDir) - - val cmd = Seq( - "java", "-jar", jarPath.getAbsolutePath, - "report", xml.getAbsolutePath, - "--classfiles", classFilesDir.getAbsolutePath, - "--html", reportDir.getAbsolutePath - ) - log.info(s"cmd: ${cmd.mkString(" ")}") - val exitCode = Process(cmd, baseDir).! - if (exitCode != 0) sys.error("JaCoCo CLI report generation failed") - - log.info(s"Filtered HTML report generated at: ${reportDir.getAbsolutePath}") - } - } - } } diff --git a/project/plugins.sbt b/project/plugins.sbt index c0fb160b..3066af24 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -30,23 +30,21 @@ addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.2.0") -addSbtPlugin("com.github.sbt" % "sbt-jacoco" % "3.5.0") - // sbt-jacoco dependency downloading -//lazy val ow2Version = "9.5" -//lazy val jacocoVersion = "0.8.11-absa.1" -// -//def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" -//def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" -// -//addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") -//addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") -// -//addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) -//addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) -//addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) -//addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) -//addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) -//addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) -// -//addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") +lazy val ow2Version = "9.5" +lazy val jacocoVersion = "0.8.11-absa.1" + +def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" +def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" + +addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") +addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") + +addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) +addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) +addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) +addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) +addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) +addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) + +addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") From 44a803bef356a6c4b28c8b1a1a7abee59385789c Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 26 Jun 2025 15:17:35 +0200 Subject: [PATCH 21/38] Update agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala --- agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala index beddf18f..8e9dba60 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala @@ -27,8 +27,6 @@ import za.co.absa.atum.model.types.basic.AtumPartitionsOps */ trait AtumAgent { - - private[this] var contexts: Map[AtumPartitions, AtumContext] = Map.empty val dispatcher: Dispatcher From e481938d827538fc97665f95e5cc31583af11a9a Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 26 Jun 2025 15:20:19 +0200 Subject: [PATCH 22/38] Minor fixes and cleanup. --- .github/workflows/jacoco_report.yml | 6 ++---- build.sbt | 4 ++-- project/JacocoSetup.scala | 3 --- project/plugins.sbt | 1 - 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index c2f3e9c4..bf4df44a 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -73,14 +73,12 @@ jobs: - name: Build and run tests with test coverage continue-on-error: true id: jacocorun - run: | - sbt jacoco + run: sbt jacoco - name: Filter jacoco xml files continue-on-error: true id: jacocofilter - run: | - sbt filterJacoco + run: sbt filterJacoco - name: Check coverage thresholds and add reports in PR comments id: jacoco diff --git a/build.sbt b/build.sbt index 5db8e98e..6c7f977b 100644 --- a/build.sbt +++ b/build.sbt @@ -14,11 +14,11 @@ * limitations under the License. */ +import sbt.* +import sbt.Keys.* import Dependencies.* import Dependencies.Versions.spark3 import VersionAxes.* -import sbt.* -import sbt.Keys.* ThisBuild / scalaVersion := Setup.scala213.asString diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index 009e3189..2e088e4e 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -25,8 +25,6 @@ import java.nio.file.{Files, StandardCopyOption} object JacocoSetup { - private val JacocoCLI = config("jacococli").hide - private val jacocoReportCommonSettings: JacocoReportSettings = JacocoReportSettings( formats = Seq(JacocoReportFormats.HTML, JacocoReportFormats.XML) ) @@ -58,7 +56,6 @@ object JacocoSetup { ) } - // Injectable task def filterJacocoTask: Def.Initialize[Task[Unit]] = Def.task { val log = streams.value.log val os = sys.props("os.name").toLowerCase diff --git a/project/plugins.sbt b/project/plugins.sbt index 3066af24..80117415 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -29,7 +29,6 @@ addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") // Plugins to build the server module as a jar file addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.2.0") - // sbt-jacoco dependency downloading lazy val ow2Version = "9.5" lazy val jacocoVersion = "0.8.11-absa.1" From 93c0a637a1bbca787be49e42ef9b232db01f6c40 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Mon, 1 Sep 2025 14:42:50 +0200 Subject: [PATCH 23/38] Initial removal of the old Jacoco solution. Temporal disable of jacoco CI run. --- .github/workflows/jacoco_report.yml | 7 +- README.md | 46 +--------- build.sbt | 7 -- project/JacocoSetup.scala | 129 ---------------------------- project/plugins.sbt | 19 ---- 5 files changed, 6 insertions(+), 202 deletions(-) delete mode 100644 project/JacocoSetup.scala diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index bf4df44a..32200274 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -17,9 +17,10 @@ name: JaCoCo Report on: - pull_request: - branches: [ master ] - types: [ opened, edited, synchronize, reopened ] + workflow_dispatch: +# pull_request: +# branches: [ master ] +# types: [ opened, edited, synchronize, reopened ] env: scalaLong: 2.13.11 diff --git a/README.md b/README.md index 211ddaf0..c52ed363 100644 --- a/README.md +++ b/README.md @@ -239,57 +239,15 @@ sbt jacoco The HTML report of coverage will be generated on the path: ``` -{project-root}/{module}/target/jvm-{scala_version}/jacoco/report/html/index.html +{project-root}/{module}/target/jacoco/report/index.html ``` The XML report of coverage will be generated on the path: ``` -{project-root}/{module}/target/jvm-{scala_version}/jacoco/report/jacoco.xml +{project-root}/{module}/target/jacoco/report/jacoco.xml ``` -### Exclusion During Coverage Report Generation - -To filter out files from the code coverage report, you can define their sequence in the file `{project-root}/project/JacocoSetup.scala`. - -``` - def jacocoProjectExcludes(): Seq[String] = { - Seq( - "**.api.http.*", - "**.config.*", - "za.co.absa.atum.agent.dispatcher.Dispatcher", - "za.co.absa.atum.agent.dispatcher.ConsoleDispatcher", - "za.co.absa.atum.server.Main*", - "za.co.absa.atum.server.Constants*", - "za.co.absa.atum.server.api.database.DoobieImplicits*", - "za.co.absa.atum.server.api.database.TransactorProvider*", - "za.co.absa.atum.model.envelopes.Pagination", - "za.co.absa.atum.model.envelopes.ResponseEnvelope", - "za.co.absa.atum.model.envelopes.StatusResponse", - "za.co.absa.atum.model.envelopes.SuccessResponse" - ) - } -``` - -### Exclusion After Coverage Report Generation - -To filter out files, objects (`class`, `object`, `trait`), and methods from the code coverage report after it has been generated, you can define their sequence in the file `{project-root}/jacoco_filter.toml`. - -``` -rules = [ - # filter-out methods from AtumAgent class - "method:za.co.absa.atum.agent.AtumAgent#dispatcherFromConfig*", - - # filter-out methods from AtumAgent object - "method:za.co.absa.atum.agent.AtumAgent$#dispatcherFromConfig*" -] -``` - -> For filtration, a [Jacoco Filter](https://github.com/MoranaApps/jacoco-filter) tool is used. ->
**Hint:** To identify object names, observe values in the jacoco.xml files with non-zero missed instructions. - -> **Important:** In the used version of the tool, the filtration is applied on the XML report, not on the HTML report! - ## How to Run in IntelliJ To make this project runnable via IntelliJ, do the following: diff --git a/build.sbt b/build.sbt index 6c7f977b..97db2098 100644 --- a/build.sbt +++ b/build.sbt @@ -141,10 +141,3 @@ lazy val reader = (projectMatrix in file("reader")) ) .addScalaCrossBuild(Setup.clientSupportedScalaVersions, Dependencies.readerDependencies) .dependsOn(model) - -/** - * Register a Jacoco filter task that runs the Jacoco-filter script on JaCoCo XML files. - * This task expects a configuration file 'jacoco_filter.toml' to be present in the project root directory. - */ -lazy val filterJacoco = taskKey[Unit]("Run jacoco-filter on JaCoCo XML files") -filterJacoco := JacocoSetup.filterJacocoTask.value diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala deleted file mode 100644 index df7a85a8..00000000 --- a/project/JacocoSetup.scala +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import com.github.sbt.jacoco.JacocoKeys.JacocoReportFormats -import com.github.sbt.jacoco.report.JacocoReportSettings -import sbt.* -import sbt.Keys.* - -import scala.sys.process.* -import za.co.absa.commons.version.Version - -import java.nio.file.{Files, StandardCopyOption} - -object JacocoSetup { - - private val jacocoReportCommonSettings: JacocoReportSettings = JacocoReportSettings( - formats = Seq(JacocoReportFormats.HTML, JacocoReportFormats.XML) - ) - - def jacocoSettings(sparkVersion: String, scalaVersion: Version, moduleName: String): JacocoReportSettings = { - jacocoReportCommonSettings.withTitle( - s"Jacoco Report on `$moduleName` for spark:$sparkVersion - scala:${scalaVersion.asString}" - ) - } - - def jacocoSettings(scalaVersion: Version, moduleName: String): JacocoReportSettings = { - jacocoReportCommonSettings.withTitle(s"Jacoco Report on `$moduleName` for scala:${scalaVersion.asString}") - } - - def jacocoProjectExcludes(): Seq[String] = { - Seq( - "**.api.http.*", - "**.config.*", - "za.co.absa.atum.agent.dispatcher.Dispatcher", - "za.co.absa.atum.agent.dispatcher.ConsoleDispatcher", - "za.co.absa.atum.server.Main*", - "za.co.absa.atum.server.Constants*", - "za.co.absa.atum.server.api.database.DoobieImplicits*", - "za.co.absa.atum.server.api.database.TransactorProvider*", - "za.co.absa.atum.server.api.common.http.Routes*", - "za.co.absa.atum.server.api.v2.repository.PartitioningRepository", - "za.co.absa.atum.server.api.v2.repository.PartitioningRepository$", - "za.co.absa.atum.server.api.v2.controller.PartitioningController", - "za.co.absa.atum.server.api.v2.controller.PartitioningController$", - "za.co.absa.atum.server.api.v2.service.PartitioningService", - "za.co.absa.atum.server.api.v2.service.PartitioningService$", - "za.co.absa.atum.server.api.common.http.Routes*", - "za.co.absa.atum.server.implicits.SeqImplicits*", - "za.co.absa.atum.model.envelopes.Pagination", - "za.co.absa.atum.model.dto.PartitioningParentPatchDTO*", - "za.co.absa.atum.model.ApiPaths*", - "za.co.absa.atum.model.envelopes.ResponseEnvelope", - "za.co.absa.atum.model.envelopes.StatusResponse", - "za.co.absa.atum.model.envelopes.SuccessResponse" - - ) - } - - def filterJacocoTask: Def.Initialize[Task[Unit]] = Def.task { - val log = streams.value.log - val os = sys.props("os.name").toLowerCase - val isWindows = os.contains("win") - val isMac = os.contains("mac") - val isLinux = os.contains("linux") - - val zipName = if (isWindows) { - "jacoco-filter-Windows.zip" - } else if (isMac) { - "jacoco-filter-macOS.zip" - } else if (isLinux) { - "jacoco-filter-Linux.zip" - } else { - sys.error(s"Unsupported OS: $os") - } - - val downloadUrl = s"https://github.com/MoranaApps/jacoco-filter/releases/latest/download/$zipName" - val targetDir = target.value - val zipPath = targetDir / zipName - val binaryName = zipName.replace(".zip", "") - val binaryPath = targetDir / binaryName / "jacoco-filter" / binaryName.toLowerCase() - - if (!binaryPath.exists()) { - log.info(s"Downloading $zipName...") - - val connection = new URL(downloadUrl).openConnection() - val inputStream = connection.getInputStream - Files.copy(inputStream, zipPath.toPath, StandardCopyOption.REPLACE_EXISTING) - inputStream.close() - - log.info("Unzipping binary...") - IO.unzip(zipPath, targetDir) - - if (!isWindows) { - binaryPath.setExecutable(true) - } - } else { - log.info(s"Binary already exists: $binaryPath") - } - - val tomlPath = baseDirectory.value / "jacoco_filter.toml" - if (!tomlPath.exists()) { - sys.error(s"Config file not found: $tomlPath") - } - - log.info(s"Running jacoco-filter with config: $tomlPath") - - val cmd = if (isWindows) - Seq("cmd", "/c", binaryPath.getAbsolutePath, "--config", tomlPath.getAbsolutePath) - else - Seq(binaryPath.getAbsolutePath, "--config", tomlPath.getAbsolutePath) - - val exitCode = Process(cmd).! - - if (exitCode != 0) - sys.error(s"jacoco-filter failed with exit code $exitCode") - } -} diff --git a/project/plugins.sbt b/project/plugins.sbt index 80117415..f62b477b 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -28,22 +28,3 @@ addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") // Plugins to build the server module as a jar file addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.2.0") - -// sbt-jacoco dependency downloading -lazy val ow2Version = "9.5" -lazy val jacocoVersion = "0.8.11-absa.1" - -def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" -def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" - -addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") -addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") - -addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) -addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) -addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) -addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) -addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) -addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) - -addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") From 92ba25e8063fb7363140bdd0e0abf22383640256 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Mon, 1 Sep 2025 14:46:53 +0200 Subject: [PATCH 24/38] Comment out jacoco from project. --- project/Setup.scala | 4 ++-- project/VersionAxes.scala | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/project/Setup.scala b/project/Setup.scala index f986192f..72057e8c 100644 --- a/project/Setup.scala +++ b/project/Setup.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -import com.github.sbt.jacoco.JacocoKeys.jacocoExcludes +//import com.github.sbt.jacoco.JacocoKeys.jacocoExcludes import sbt.* import sbt.Keys.* import sbtassembly.AssemblyKeys.assemblyMergeStrategy @@ -37,7 +37,7 @@ object Setup { lazy val commonSettings: Seq[SettingsDefinition] = Seq( scalacOptions ++= Setup.commonScalacOptions, Test / parallelExecution := false, - jacocoExcludes := JacocoSetup.jacocoProjectExcludes(), +// jacocoExcludes := JacocoSetup.jacocoProjectExcludes(), (assembly / test) := {}, (publish / test) := { (Test / testOnly).toTask(" *UnitTests").value } ) diff --git a/project/VersionAxes.scala b/project/VersionAxes.scala index a52aec46..9d3e319c 100644 --- a/project/VersionAxes.scala +++ b/project/VersionAxes.scala @@ -16,8 +16,8 @@ import sbt.{Def, ModuleID, VirtualAxis, *} import sbt.Keys.* import sbt.internal.ProjectMatrix -import JacocoSetup.jacocoSettings -import com.github.sbt.jacoco.JacocoKeys.jacocoReportSettings +//import JacocoSetup.jacocoSettings +//import com.github.sbt.jacoco.JacocoKeys.jacocoReportSettings import za.co.absa.commons.version.Version @@ -55,7 +55,7 @@ object VersionAxes { libraryDependencies ++= dependenciesFnc(sparkVersion, scalaVersion), printVersionInfo := streams.value.log.info(s"Building ${name.value} with Spark $sparkVersion, Scala ${scalaVersion.asString}"), (Compile / compile) := ((Compile / compile) dependsOn printVersionInfo).value, - jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion, name.value), +// jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion, name.value), ).settings(settings *) ) } @@ -73,7 +73,7 @@ object VersionAxes { libraryDependencies ++= dependenciesFnc(scalaVersion), printVersionInfo := streams.value.log.info(s"Building ${name.value} with Scala ${scalaVersion.asString}"), (Compile / compile) := ((Compile / compile) dependsOn printVersionInfo).value, - jacocoReportSettings := jacocoSettings(scalaVersion, name.value), +// jacocoReportSettings := jacocoSettings(scalaVersion, name.value), ).settings(settings *) ) ) @@ -90,7 +90,7 @@ object VersionAxes { scalacOptions ++= Setup.serverAndDbScalacOptions, printVersionInfo := streams.value.log.info(s"Building ${name.value} with Scala ${scalaVersion.asString}"), (Compile / compile) := ((Compile / compile) dependsOn printVersionInfo).value, - jacocoReportSettings := jacocoSettings(scalaVersion, name.value), +// jacocoReportSettings := jacocoSettings(scalaVersion, name.value), ).settings(settings *) ) } From d106bb404a4b7339e558926cf25cd70eba328d1d Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Mon, 1 Sep 2025 15:02:41 +0200 Subject: [PATCH 25/38] Introduced auto-plugin. Plugin enabled for project modules. --- build.sbt | 5 + project/FilteredJacocoAgentPlugin.scala | 333 ++++++++++++++++++++++++ project/JacocoBaseKeysPlugin.scala | 26 ++ 3 files changed, 364 insertions(+) create mode 100644 project/FilteredJacocoAgentPlugin.scala create mode 100644 project/JacocoBaseKeysPlugin.scala diff --git a/build.sbt b/build.sbt index 97db2098..5c068d1b 100644 --- a/build.sbt +++ b/build.sbt @@ -69,6 +69,7 @@ lazy val server = { ) .enablePlugins(AssemblyPlugin) .enablePlugins(AutomateHeaderPlugin) + .enablePlugins(FilteredJacocoAgentPlugin) .addSingleScalaBuild(Setup.serverAndDbScalaVersion, Dependencies.serverDependencies) .dependsOn(model) @@ -90,6 +91,7 @@ lazy val agent = (projectMatrix in file("agent")) javacOptions ++= Setup.clientJavacOptions ): _* ) + .enablePlugins(FilteredJacocoAgentPlugin) .addSparkCrossBuild(SparkVersionAxis(spark3), Setup.clientSupportedScalaVersions, Dependencies.agentDependencies) .dependsOn(model) @@ -104,6 +106,7 @@ lazy val model = (projectMatrix in file("model")) javacOptions ++= Setup.clientJavacOptions, ): _* ) + .enablePlugins(FilteredJacocoAgentPlugin) .addScalaCrossBuild(Setup.clientSupportedScalaVersions, Dependencies.modelDependencies) /** @@ -119,6 +122,7 @@ lazy val database = { publish / skip := true ): _* ) + .enablePlugins(FilteredJacocoAgentPlugin) .addSingleScalaBuild(Setup.serverAndDbScalaVersion, Dependencies.databaseDependencies) if (limitedProject) { null // if value other then null is returned, the condition doesn't seem to work. @@ -139,5 +143,6 @@ lazy val reader = (projectMatrix in file("reader")) javacOptions ++= Setup.clientJavacOptions ): _* ) + .enablePlugins(FilteredJacocoAgentPlugin) .addScalaCrossBuild(Setup.clientSupportedScalaVersions, Dependencies.readerDependencies) .dependsOn(model) diff --git a/project/FilteredJacocoAgentPlugin.scala b/project/FilteredJacocoAgentPlugin.scala new file mode 100644 index 00000000..b0f6f40d --- /dev/null +++ b/project/FilteredJacocoAgentPlugin.scala @@ -0,0 +1,333 @@ +import JacocoBaseKeysPlugin.autoImport.* +import sbt.* +import sbt.Keys.* + +/** + * JacocoAgentPlugin (no aggregation/merge) + * --------------------------------------- + * - Attaches JaCoCo agent to forked JVMs per module (Test + optional IntegrationTest) + * - Writes per-module .exec files (no merging) + * - Generates per-module reports + * - Provides root helpers: jacocoCleanAll / jacocoReportAll that just iterate modules (no merge) + */ +object FilteredJacocoAgentPlugin extends AutoPlugin { + object autoImport { + val jacocoVersion = settingKey[String]("JaCoCo version") + val jacocoExecFile = settingKey[File]("Per-module JaCoCo .exec file (Test)") + val jacocoItExecFile = settingKey[File]("Per-module JaCoCo .exec file (IntegrationTest)") + val jacocoReportDir = settingKey[File]("Per-module report directory") + val jacocoIncludes = settingKey[Seq[String]]("Include patterns (JaCoCo syntax)") + val jacocoExcludes = settingKey[Seq[String]]("Exclude patterns (JaCoCo syntax)") + val jacocoAppend = settingKey[Boolean]("Append to existing .exec instead of overwrite (default: false)") + val jacocoFailOnMissingExec = + settingKey[Boolean]("Fail jacocoReport if .exec is missing (default: false – warn & skip)") + + val jacocoReportName = settingKey[String]("Title used for JaCoCo HTML report") + + // Root-only helpers (NO MERGE): just run per-module tasks across aggregated projects + val jacocoCleanAll = taskKey[Unit]("Run jacocoClean in all aggregated modules (no merge)") + val jacocoReportAll = taskKey[Unit]("Run jacocoReport in all aggregated modules (no merge)") + + val jacocoSetUserDirToBuildRoot = settingKey[Boolean]("Mimic non-forked runs by setting -Duser.dir to the build root for forked tests") + + val jmfCoreVersion = settingKey[String]("JMF core library version") + val Jmf = config("jmf").hide + val jmfRewrite = taskKey[File]("Rewrite compiled classes using JMF tool; returns output dir") + val jmfOutDir = settingKey[File]("JMF output base dir") + val jmfRulesFile = settingKey[File]("JMF rules file") + val jmfCliMain = settingKey[String]("Main class of the JMF CLI") + val jmfDryRun = settingKey[Boolean]("Dry-run rewriter") + val jmfEnabled = settingKey[Boolean]("Enable JMF rewriting") + val jmfPrepareForTests = taskKey[Unit]("Run JMF rewrite when enabled (no self-ref to test)") + } + import autoImport.* + + override def requires = JacocoBaseKeysPlugin + override def trigger = noTrigger + + // ---- helper: all aggregated descendants (BFS), excluding the root itself + private def aggregatedDescendants(e: Extracted, root: ProjectRef): Vector[ProjectRef] = { + val s = e.structure + val seen = scala.collection.mutable.LinkedHashSet[ProjectRef](root) + val queue = scala.collection.mutable.Queue[ProjectRef](root) + while (queue.nonEmpty) { + val ref = queue.dequeue() + val kids = Project.getProject(ref, s).toList.flatMap(_.aggregate) + kids.foreach { k => if (!seen(k)) { seen += k; queue.enqueue(k) } } + } + seen.toVector.tail // drop root + } + + // ---- helper: only those that set jacocoPluginEnabled := true + private def enabledUnder(state: State): Vector[ProjectRef] = { + val e = Project.extract(state) + val here = e.currentRef + val all = aggregatedDescendants(e, here) // children only (no root) + all.filter { ref => + e.getOpt((ref / jacocoPluginEnabled): SettingKey[Boolean]).getOrElse(false) + } + } + + // ---- commands + private lazy val jacocoCleanAllCmd = Command.command("jacocoCleanAll") { state => + val targets = enabledUnder(state) + if (targets.isEmpty) { println("[jacoco] nothing to clean (no enabled modules under this aggregate)."); state } + else targets.foldLeft(state) { (st, ref) => Command.process(s"${ref.project}/jacocoClean", st) } + } + + private lazy val jacocoReportAllCmd = Command.command("jacocoReportAll") { state => + val targets = enabledUnder(state) + if (targets.isEmpty) { println("[jacoco] nothing to report (no enabled modules under this aggregate)."); state } + else targets.foldLeft(state) { (st, ref) => Command.process(s"${ref.project}/jacocoReport", st) } + } + + // ---- global defaults so keys exist everywhere (safe no-ops on projects without the plugin) + override def buildSettings: Seq[Def.Setting[_]] = Seq( + jacocoPluginEnabled := false, // overridden to true in projects that enable the plugin + // register commands + a convenient alias like sbt-jacoco had + commands ++= Seq(jacocoCleanAllCmd, jacocoReportAllCmd) + ) + + private def findOnCp(cp: Seq[Attributed[File]])(p: File => Boolean): Option[File] = + cp.map(_.data).find(p) + + private def agentJar(cp: Seq[Attributed[File]]): File = { + val files = cp.map(_.data) + files.find(f => f.getName.startsWith("org.jacoco.agent-") && f.getName.contains("-runtime")) + .orElse(files.find(f => f.getName.contains("jacoco") && f.getName.contains("agent") && f.getName.contains("runtime"))) + .orElse(files.find(f => f.getName.startsWith("org.jacoco.agent-") && f.getName.endsWith(".jar"))) // last resort + .getOrElse(sys.error("JaCoCo runtime agent JAR not found on Test / dependencyClasspath")) + } + + private def cliJar(cp: Seq[Attributed[File]]): File = { + val files = cp.map(_.data) + files.find(f => f.getName.startsWith("org.jacoco.cli-") && f.getName.contains("nodeps")) + .orElse(files.find(_.getName.startsWith("org.jacoco.cli-"))) // fallback, but we won't use it + .getOrElse(sys.error("org.jacoco.cli (nodeps) JAR not found on Test / dependencyClasspath")) + } + + private val defaultIncludes = Seq("**") + private val defaultExcludes = Seq("scala.*", "java.*", "sun.*", "jdk.*") + + override def projectSettings: Seq[Setting[_]] = Seq( + jacocoPluginEnabled := false, + + // ---- coordinates + jacocoVersion := "0.8.12", + jmfCoreVersion := "0.1.7", + libraryDependencies ++= Seq( + // pull the agent with the runtime classifier (this is the actual -javaagent jar) + ("org.jacoco" % "org.jacoco.agent" % jacocoVersion.value % Test).classifier("runtime"), + ("org.jacoco" % "org.jacoco.cli" % jacocoVersion.value % Test).classifier("nodeps"), + "io.github.moranaapps" % "jacoco-method-filter-core_2.12" % jmfCoreVersion.value % Jmf.name, + ), + jacocoSetUserDirToBuildRoot := true, + + // ---- defaults + jacocoExecFile := target.value / "jacoco" / "jacoco.exec", + jacocoReportDir := target.value / "jacoco" / "report", + jacocoIncludes := defaultIncludes, + jacocoExcludes := defaultExcludes, + jacocoAppend := false, + jacocoFailOnMissingExec := false, + + jacocoReportName := { + val moduleId = thisProject.value.id // or: thisProjectRef.value.project + s"Report: $moduleId - scala:${scalaVersion.value}" + }, + + // --- JMF tool wiring + ivyConfigurations += Jmf, + + jmfOutDir := target.value / "jmf", + jmfRulesFile:= (ThisBuild / baseDirectory).value / "jmf-rules.txt", + jmfCliMain := "io.moranaapps.jacocomethodfilter.CoverageRewriter", + jmfDryRun := false, + jmfEnabled := true, + + // the rewrite task (your code, lightly cleaned) + jmfRewrite := { + val log = streams.value.log + val enabled = jacocoPluginEnabled.value + + // ensure classes exist (safe to always do; test would compile anyway) + val _ = (Compile / compile).value + + val classesIn = (Compile / classDirectory).value + val rules = jmfRulesFile.value + val upd = (Jmf / update).value // hoisted + + if (!enabled) classesIn + else if (!classesIn.exists) { + log.warn(s"[jmf] compiled classes dir not found, skipping: ${classesIn.getAbsolutePath}"); classesIn + } else { + val hasClasses = (classesIn ** sbt.GlobFilter("*.class")).get.nonEmpty + if (!hasClasses) { log.warn(s"[jmf] no .class files under ${classesIn.getAbsolutePath}; skipping."); classesIn } + else if (!rules.exists) { log.warn(s"[jmf] rules file missing: ${rules.getAbsolutePath}; skipping."); classesIn } + else { + val outDir = jmfOutDir.value / "classes-filtered" + IO.delete(outDir); IO.createDirectory(outDir) + + val toolJars = upd.matching(artifactFilter(`type` = "jar")).distinct + log.info("[jmf] tool CP:\n" + toolJars.map(f => s" - ${f.getAbsolutePath}").mkString("\n")) + + val cpStr = toolJars.mkString(java.io.File.pathSeparator) + val args = Seq("java","-cp", cpStr, jmfCliMain.value, + "--in", classesIn.getAbsolutePath, + "--out", outDir.getAbsolutePath, + "--rules", rules.getAbsolutePath) ++ + (if (jmfDryRun.value) Seq("--dry-run") else Seq()) + + log.info(s"[jmf] rewrite: ${args.mkString(" ")}") + val code = scala.sys.process.Process(args, baseDirectory.value).! + if (code != 0) sys.error(s"[jmf] rewriter failed ($code)") + outDir + } + } + }, + + // 1) preparatory task (already defined earlier) + jmfPrepareForTests := Def.taskDyn { + if (jmfEnabled.value) Def.task { jmfRewrite.value; () } + else Def.task { () } + }.value, + + Test / fullClasspath := Def.taskDyn { + // Gather the usual ingredients + val testOut = (Test / classDirectory).value // test classes dir + val mainOut = (Compile / classDirectory).value // original main classes dir + val deps = (Test / internalDependencyClasspath).value + val ext = (Test / externalDependencyClasspath).value + val unmanaged = (Test / unmanagedClasspath).value + val scalaJars = (Test / scalaInstance).value.allJars.map(Attributed.blank(_)).toVector + val resources = (Test / resourceDirectories).value.map(Attributed.blank) + + def build(rewrittenOpt: Option[File]) = Def.task { + val rewrittenDifferent = rewrittenOpt.filter(_ != mainOut) + val prefix = rewrittenDifferent.toVector.map(Attributed.blank) :+ Attributed.blank(testOut) + val rest = (deps ++ ext ++ scalaJars ++ resources ++ unmanaged) + .filterNot(a => a.data == mainOut || a.data == testOut || rewrittenDifferent.exists(_ == a.data)) + (prefix ++ rest :+ Attributed.blank(mainOut)) + } + + if (jacocoPluginEnabled.value) build(Some(jmfRewrite.value)) + else build(None) + }.value, + + // ---- fork so -javaagent is applied + Test / fork := true, + + // Attach agent for Test + Test / forkOptions := { + val fo0 = (Test / forkOptions).value + val rootDir = (LocalRootProject / baseDirectory).value + val baseFO = fo0.withWorkingDirectory(rootDir) // keep tests running from repo root + + // pre-compute values (avoids sbt linter warning about .value inside if) + val cp = (Test / dependencyClasspath).value + val agent = agentJar(cp) + val dest = jacocoExecFile.value.getAbsolutePath + val inc = jacocoIncludes.value.mkString(":") + val exc = jacocoExcludes.value.mkString(":") + val append = if (jacocoAppend.value) "true" else "false" + val agentOpt = + s"-javaagent:${agent.getAbsolutePath}=destfile=$dest,append=$append,output=file,includes=$inc,excludes=$exc,inclbootstrapclasses=false,jmx=false" + + val log = streams.value.log + log.info(s"[jacoco] setting fork working dir to: $rootDir") + + if (jacocoPluginEnabled.value) { + log.info(s"[jacoco] agent jar: ${agent.getName} (enabled)") + baseFO.withRunJVMOptions(baseFO.runJVMOptions :+ agentOpt) + } else { + log.info("[jacoco] disabled (jacocoPluginEnabled=false); NOT adding -javaagent") + baseFO + } + }, + + // Print one sanity line per test fork + Test / testOptions += Tests.Setup { () => + val status = + try { + val rt = Class.forName("org.jacoco.agent.rt.RT") + val m = rt.getMethod("getAgent") + m.invoke(null) // throws if not attached + "attached" + } catch { + case _: ClassNotFoundException => "rt-jar-not-on-classpath" + case _: Throwable => "present-but-not-attached" + } + println(s"[jacoco] agent status: $status; user.dir=" + System.getProperty("user.dir")) + }, + + + // ---- per-module clean + jacocoClean := { + val log = streams.value.log + val outDir = target.value / "jacoco" + IO.delete(outDir) + IO.createDirectory(outDir) + IO.delete(jmfOutDir.value) + + // remove sbt-jacoco leftovers if they ever existed + val instrDir = (Test / crossTarget).value / "jacoco" / "instrumented-classes" + if (instrDir.exists) { + log.info(s"[jacoco] removing sbt-jacoco leftovers: ${instrDir.getAbsolutePath}") + IO.delete(instrDir) + } + log.info(s"[jacoco] cleaned: ${outDir.getAbsolutePath}") + }, + + // ---- per-module report (only this module, no merge) + jacocoReport := { + val log = streams.value.log + val execFile = jacocoExecFile.value + val reportDir = jacocoReportDir.value + IO.createDirectory(reportDir) + + // PRE-compute (avoid linter warnings) + val moduleName = name.value + val baseDir = baseDirectory.value + val failOnMiss = jacocoFailOnMissingExec.value + val cp = (Test / dependencyClasspath).value + val cli = cliJar(cp) + val title = jacocoReportName.value + + // Class dirs (filter to existing) + val classesIn = (Compile / classDirectory).value + val filteredDir = jmfOutDir.value / "classes-filtered" + val mainClasses = + if (jacocoPluginEnabled.value && filteredDir.exists) filteredDir + else classesIn + + val classDirs = Seq(mainClasses).filter(_.exists) + + // Source dirs: unmanaged + managed (filter to existing) + val unmanagedSrc = (Compile / unmanagedSourceDirectories).value + val managedSrc = (Compile / managedSourceDirectories).value + val srcDirs = (unmanagedSrc ++ managedSrc).filter(_.exists) + + if (!execFile.exists) { + val msg = s"[jacoco] .exec not found for $moduleName: $execFile . Run tests first." + if (failOnMiss) sys.error(msg) else { log.warn(msg); reportDir } + } else if (classDirs.isEmpty) { + log.warn(s"[jacoco] no class dirs for $moduleName; skipping report.") + reportDir + } else { + // repeat flags per path + val args = Seq("java","-jar", cli.getAbsolutePath, "report", execFile.getAbsolutePath) ++ + classDirs.flatMap(d => Seq("--classfiles", d.getAbsolutePath)) ++ + srcDirs .flatMap(d => Seq("--sourcefiles", d.getAbsolutePath)) ++ + Seq("--name", title, + "--html", reportDir.getAbsolutePath, + "--xml", (reportDir / "jacoco.xml").getAbsolutePath, + "--csv", (reportDir / "jacoco.csv").getAbsolutePath) + + val exit = scala.sys.process.Process(args, baseDir).! + if (exit != 0) sys.error("JaCoCo report generation failed") + log.info(s"[jacoco] per-module HTML: ${reportDir / "index.html"}") + reportDir + } + } + ) +} diff --git a/project/JacocoBaseKeysPlugin.scala b/project/JacocoBaseKeysPlugin.scala new file mode 100644 index 00000000..cd2ab0a3 --- /dev/null +++ b/project/JacocoBaseKeysPlugin.scala @@ -0,0 +1,26 @@ +import sbt.* +import sbt.Keys.* + +object JacocoBaseKeysPlugin extends AutoPlugin { + object autoImport { + val jacocoPluginEnabled = settingKey[Boolean]("Marker for JaCoCo plugin participation") + val jacocoClean = taskKey[Unit]("Clean JaCoCo outputs") + val jacocoReport = taskKey[File]("Generate per-module JaCoCo report") + } + import autoImport.* + + // apply to every project (project scope → target.value etc. are valid) + override def trigger = allRequirements + override def requires = plugins.JvmPlugin + + override def projectSettings: Seq[Def.Setting[_]] = Seq( + jacocoPluginEnabled := false, // default: not participating + jacocoClean := { streams.value.log.debug("[jacoco] not enabled here; clean no-op.") }, + jacocoReport := { + val d = target.value / "jacoco" / "report" // safe placeholder dir + IO.createDirectory(d) + streams.value.log.debug("[jacoco] not enabled here; report no-op.") + d + } + ) +} From 594d5a78a1a341ae60182ba864dcbdead4877194 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Mon, 1 Sep 2025 15:08:50 +0200 Subject: [PATCH 26/38] Re-activated jacoco in CI. Added rules files. Added three required aliases. --- .github/workflows/jacoco_report.yml | 12 +-- build.sbt | 5 + jmf-rules.txt | 154 ++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 9 deletions(-) create mode 100644 jmf-rules.txt diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 32200274..9080d5f7 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -17,10 +17,9 @@ name: JaCoCo Report on: - workflow_dispatch: -# pull_request: -# branches: [ master ] -# types: [ opened, edited, synchronize, reopened ] + pull_request: + branches: [ master ] + types: [ opened, edited, synchronize, reopened ] env: scalaLong: 2.13.11 @@ -76,11 +75,6 @@ jobs: id: jacocorun run: sbt jacoco - - name: Filter jacoco xml files - continue-on-error: true - id: jacocofilter - run: sbt filterJacoco - - name: Check coverage thresholds and add reports in PR comments id: jacoco uses: MoranaApps/jacoco-report@v2 diff --git a/build.sbt b/build.sbt index 5c068d1b..3db1c547 100644 --- a/build.sbt +++ b/build.sbt @@ -146,3 +146,8 @@ lazy val reader = (projectMatrix in file("reader")) .enablePlugins(FilteredJacocoAgentPlugin) .addScalaCrossBuild(Setup.clientSupportedScalaVersions, Dependencies.readerDependencies) .dependsOn(model) + +// Run activate jacoco + clean + test + per-module reports across the whole build + deactivate jacoco +addCommandAlias("jacoco", "; jacocoOn; clean; test; jacocoReportAll; jacocoOff") +addCommandAlias("jacocoOff", "; set every jacocoPluginEnabled := false") +addCommandAlias("jacocoOn", "; set every jacocoPluginEnabled := true") diff --git a/jmf-rules.txt b/jmf-rules.txt new file mode 100644 index 00000000..491d545a --- /dev/null +++ b/jmf-rules.txt @@ -0,0 +1,154 @@ +# jacoco-method-filter — Default Rules & HowTo (Scala + Java) +# +# This file defines which methods should be annotated as *Generated so JaCoCo ignores them. +# One rule per line. +# +# ───────────────────────────────────────────────────────────────────────────── +# HOW TO USE (quick) +# 1) Replace YOUR.PACKAGE.ROOT with your project’s package root (e.g., com.example.app). +# 2) Start with the CONSERVATIVE section only. +# 3) If clean, enable STANDARD. Use AGGRESSIVE only inside DTO/auto‑generated packages. +# 4) Keep rules narrow (by package), prefer flags (synthetic/bridge) for compiler artifacts, +# and add `id:` labels so logs are easy to read. +# +# ───────────────────────────────────────────────────────────────────────────── +# ALLOWED SYNTAX (cheat sheet) +# +# General form: +# #() [FLAGS and PREDICATES...] +# +# FQCN_glob (dot form; $ allowed for inner classes): +# Examples: *.model.*, com.example.*, * +# +# method_glob (glob on method name): +# Examples: copy | $anonfun$* | get* | *_$eq +# +# descriptor_glob (JVM descriptor in (args)ret). You may omit it entirely. +# • Omitting descriptor ⇒ treated as "(*)*" (any args, any return). +# • Short/empty forms "", "()", "(*)" normalize to "(*)*". +# Examples: +# (I)I # takes int, returns int +# (Ljava/lang/String;)V # takes String, returns void +# () or (*) or omitted # any args, any return +# +# FLAGS (optional) — space or comma separated: +# public | protected | private | synthetic | bridge | static | abstract +# +# PREDICATES (optional): +# ret: # match return type only (e.g., ret:V, ret:I, ret:Lcom/example/*;) +# id: # identifier shown in logs/reports +# name-contains: # method name must contain +# name-starts: # method name must start with +# name-ends: # method name must end with +# +# Notes +# - Always use dot-form (com.example.Foo) for class names. +# - Comments (# …) and blank lines are ignored. +# +# ───────────────────────────────────────────────────────────────────────────── +# QUICK EXAMPLES +# +# Simple wildcards +# *#*(*) +# → Match EVERY method in EVERY class (any package). Useful only for diagnostics. +# "(*)" normalizes to "(*)*" ⇒ any args, any return. +# *.dto.*#*(*) +# → Match every method on any class under any package segment named "dto". +# Good when you treat DTOs as generated/boilerplate. + +# Scala case class helpers +# *.model.*#copy(*) +# → Matches Scala case-class `copy` methods under `*.model.*`. +# Hides boilerplate clones with any parameter list and any return. +# *.model.*#productArity() +# → Matches zero-arg `productArity` (case-class/Product API). +# *.model.*#productElement(*) +# → Matches `productElement(int)` (or any descriptor form) on case classes. +# *.model.*#productPrefix() +# → Matches `productPrefix()`; returns the case class' constructor name. + +# Companion objects and defaults +# *.model.*$*#apply(*) +# → Matches companion `apply` factories under `*.model.*` (any args). +# BE CAREFUL: can hide real factory logic; keep the package scope narrow. +# *.model.*$*#unapply(*) +# → Matches extractor `unapply` methods in companions under `*.model.*`. +# *#*$default$*(*) +# → Matches Scala-generated default-argument helpers everywhere. +# Safe to keep enabled; they’re compiler-synthesized. + +# Anonymous / synthetic / bridge +# *#$anonfun$* +# → Matches any method whose name contains `$anonfun$` (Scala lambdas). +# Consider adding `synthetic` and/or a package scope in real configs. +# *#*(*):synthetic # any synthetic +# → Matches ANY method marked `synthetic` (compiler-generated). +# Powerful; scope by package to avoid hiding intentional glue code. +# *#*(*):bridge # any bridge +# → Matches Java generic bridge methods the compiler inserts. +# Usually safe globally, but scoping is still recommended. + +# Setters / fluent APIs +# *.dto.*#*_$eq(*) +# → Matches Scala var setters in DTO packages (e.g., `name_=(...)`). +# Good for excluding trivial field writes. +# *.builder.*#with*(*) +# → Matches builder-style fluent setters (`withXxx(...)`) in builder pkgs. +# Treats chainable configuration as boilerplate. +# *.client.*#with*(*) ret:Lcom/api/client/* +# → Like above but ONLY when the return type matches your client package. +# The `ret:` predicate protects real logic that returns other types. + +# Return-type constraints +# *.jobs.*#*(*):ret:V +# → Any method under `*.jobs.*` returning `void` (`V`). Often orchestration. +# *.math.*#*(*):ret:I +# → Any method under `*.math.*` returning primitive int (`I`). +# *.model.*#*(*):ret:Lcom/example/model/* +# → Any method under `*.model.*` that returns a type in `com.example.model`. +# Handy when the *return type* uniquely identifies boilerplate. + +# ───────────────────────────────────────────────────────────────────────────── +# GLOBALS RULES +# ───────────────────────────────────────────────────────────────────────────── +# ** all case class boilerplate + +# Scala case class helpers +*#canEqual(*) id:case-canequal +*#equals(*) id:case-equals +*#apply(*) id:case-apply +*#unapply(*) id:case-unapply +*#hashCode(*) id:case-hashcode +*#copy(*) id:case-copy +*#copy$default$*(*) id:case-copy-defaults +*#productElement() id:case-prod-element +*#productArity() id:case-prod-arity +*#productPrefix() id:case-prod-prefix +*#productIterator() id:case-prod-iterator +*#tupled() id:case-tupled +*#curried() id:case-curried +*#toString() id:case-tostring +*#name() id:case-name +*#groups() id:case-groups +*#optionalAttributes() id:case-optionalAttributes + +# Companion objects, constructors, and static definitions +*$#(*) id:gen-ctor # constructors +*$#() id:gen-clinit # static initializer blocks + +# Companion objects and defaults +*$*#apply(*) id:comp-apply +*$*#unapply(*) id:comp-unapply +*$*#toString(*) id:comp-tostring +*$*#readResolve(*) id:comp-readresolve + +# anonymous class created by a macro expansion +*$macro$*#$anonfun$inst$macro$* id:macro-inst +*$macro$*#inst$macro$* id:macro-inst + +# lambda +*#* synthetic name-contains:$anonfun$ id:scala-anonfun + +# ───────────────────────────────────────────────────────────────────────────── +# DIRECT RULES +# ───────────────────────────────────────────────────────────────────────────── From d687d92ef6046dd95c6a65b9efd0e2222e7529f3 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Mon, 1 Sep 2025 15:15:02 +0200 Subject: [PATCH 27/38] Removed database module from jacoco coverage. Added everal fake changes to test full run. --- agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala | 3 +++ build.sbt | 1 - .../scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala | 1 + reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala | 2 ++ server/src/main/scala/za/co/absa/atum/server/Main.scala | 3 +++ 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index 458c57dc..fc991d9d 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -46,6 +46,9 @@ class AtumContext private[agent] ( */ def currentMeasures: Set[AtumMeasure] = measures + + + /** * Returns the sub-partition context in the AtumContext. * diff --git a/build.sbt b/build.sbt index 3db1c547..1de7a587 100644 --- a/build.sbt +++ b/build.sbt @@ -122,7 +122,6 @@ lazy val database = { publish / skip := true ): _* ) - .enablePlugins(FilteredJacocoAgentPlugin) .addSingleScalaBuild(Setup.serverAndDbScalaVersion, Dependencies.databaseDependencies) if (limitedProject) { null // if value other then null is returned, the condition doesn't seem to work. diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala index 17ec0398..11c7ab60 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala @@ -26,4 +26,5 @@ case class AdditionalDataDTO( object AdditionalDataDTO { implicit val encodeAdditionalDataDTO: Encoder[AdditionalDataDTO] = deriveEncoder implicit val decodeAdditionalDataDTO: Decoder[AdditionalDataDTO] = deriveDecoder + } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index a07d95b8..9c660f33 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -62,6 +62,8 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit } yield checkpointsOrError } + + /** * Function to retrieve a page of checkpoints of the given name belonging to the flow. * The checkpoints are ordered by their creation order. diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index cb159391..9a2ab939 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -129,6 +129,9 @@ object Main extends ZIOAppDefault { } + + + override val bootstrap: ZLayer[Any, Config.Error, Unit] = Runtime.removeDefaultLoggers >>> SLF4J.slf4j >>> Runtime.setConfigProvider(configProvider) From 203ddfb879152a17dc80176cc16e80db4f9e12d1 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Mon, 1 Sep 2025 15:39:47 +0200 Subject: [PATCH 28/38] Fixed expected path to jacoco.xml file. --- .github/workflows/jacoco_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 9080d5f7..75d943ec 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -81,7 +81,7 @@ jobs: with: token: '${{ secrets.GITHUB_TOKEN }}' paths: | - **/target/*${{ env.scalaShort }}*/jacoco/report/jacoco.filtered.xml + **/target/jacoco/report/jacoco.xml exclude-paths: | database/target/** sensitivity: "detail" From e531e665c99e1c65ac5bcc60be440ee85e723a0b Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Wed, 3 Sep 2025 16:58:11 +0200 Subject: [PATCH 29/38] - Migrated to new jacoco solution based on agent usage. --- .github/workflows/jacoco_report.yml | 4 +- .sbtrc | 5 ++ README.md | 26 +++++++-- build.sbt | 13 ++--- jacoco_filter.toml | 11 ---- jmf-rules.txt | 26 ++++++++- project/FilteredJacocoAgentPlugin.scala | 73 +++++++++++++++++++------ project/Setup.scala | 2 - project/VersionAxes.scala | 5 -- 9 files changed, 112 insertions(+), 53 deletions(-) delete mode 100644 jacoco_filter.toml diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 75d943ec..2af2cfb2 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -73,7 +73,7 @@ jobs: - name: Build and run tests with test coverage continue-on-error: true id: jacocorun - run: sbt jacoco + run: sbt "project server" jacoco - name: Check coverage thresholds and add reports in PR comments id: jacoco @@ -81,7 +81,7 @@ jobs: with: token: '${{ secrets.GITHUB_TOKEN }}' paths: | - **/target/jacoco/report/jacoco.xml + **/target/**/jacoco/report/jacoco.xml exclude-paths: | database/target/** sensitivity: "detail" diff --git a/.sbtrc b/.sbtrc index c89544c9..5d0e43f4 100644 --- a/.sbtrc +++ b/.sbtrc @@ -33,3 +33,8 @@ alias testAll=; testOnly * # Run agent-server compatibility tests alias testCompatibility=; testOnly *CompatibilityTests + +# Jacoco Aliases +alias jacoco=; jacocoOn; +clean; +test; jacocoReportAll; jacocoOff +alias jacocoOff=; set every jacocoPluginEnabled := false +alias jacocoOn=; set every jacocoPluginEnabled := true diff --git a/README.md b/README.md index c52ed363..562b676f 100644 --- a/README.md +++ b/README.md @@ -232,22 +232,38 @@ represents all currently supported measurement types (aka measures): ## How to generate Code coverage report + +### Module `server` + +- Use java version 11. + ```sbt -sbt jacoco +sbt "project server" jacoco ``` -The HTML report of coverage will be generated on the path: +The HTML and XML reports of coverage will be generated on the path: ``` -{project-root}/{module}/target/jacoco/report/index.html +{project-root}/{module}/target/jvm-2.13/report/jacoco/report/index.html +{project-root}/{module}/target/jvm-2.13/report/jacoco/report/jacoco.xml +``` +### Other modules + +- Use java version 8. + +```sbt +sbt jacoco ``` -The XML report of coverage will be generated on the path: +The HTML and XML reports of coverage will be generated on the path: ``` -{project-root}/{module}/target/jacoco/report/jacoco.xml +{project-root}/{module}/target/**/jacoco/report/index.html +{project-root}/{module}/target/**/jacoco/report/jacoco.xml ``` +> `**` - depends on the module setup + ## How to Run in IntelliJ To make this project runnable via IntelliJ, do the following: diff --git a/build.sbt b/build.sbt index 1de7a587..1a5adda8 100644 --- a/build.sbt +++ b/build.sbt @@ -69,8 +69,8 @@ lazy val server = { ) .enablePlugins(AssemblyPlugin) .enablePlugins(AutomateHeaderPlugin) - .enablePlugins(FilteredJacocoAgentPlugin) .addSingleScalaBuild(Setup.serverAndDbScalaVersion, Dependencies.serverDependencies) + .enablePlugins(FilteredJacocoAgentPlugin) .dependsOn(model) if (limitedProject) { @@ -91,8 +91,8 @@ lazy val agent = (projectMatrix in file("agent")) javacOptions ++= Setup.clientJavacOptions ): _* ) - .enablePlugins(FilteredJacocoAgentPlugin) .addSparkCrossBuild(SparkVersionAxis(spark3), Setup.clientSupportedScalaVersions, Dependencies.agentDependencies) + .enablePlugins(FilteredJacocoAgentPlugin) .dependsOn(model) /** @@ -106,8 +106,8 @@ lazy val model = (projectMatrix in file("model")) javacOptions ++= Setup.clientJavacOptions, ): _* ) - .enablePlugins(FilteredJacocoAgentPlugin) .addScalaCrossBuild(Setup.clientSupportedScalaVersions, Dependencies.modelDependencies) + .enablePlugins(FilteredJacocoAgentPlugin) /** * Module `database` is the source of database structures of the service @@ -142,11 +142,6 @@ lazy val reader = (projectMatrix in file("reader")) javacOptions ++= Setup.clientJavacOptions ): _* ) - .enablePlugins(FilteredJacocoAgentPlugin) .addScalaCrossBuild(Setup.clientSupportedScalaVersions, Dependencies.readerDependencies) + .enablePlugins(FilteredJacocoAgentPlugin) .dependsOn(model) - -// Run activate jacoco + clean + test + per-module reports across the whole build + deactivate jacoco -addCommandAlias("jacoco", "; jacocoOn; clean; test; jacocoReportAll; jacocoOff") -addCommandAlias("jacocoOff", "; set every jacocoPluginEnabled := false") -addCommandAlias("jacocoOn", "; set every jacocoPluginEnabled := true") diff --git a/jacoco_filter.toml b/jacoco_filter.toml deleted file mode 100644 index 4198696b..00000000 --- a/jacoco_filter.toml +++ /dev/null @@ -1,11 +0,0 @@ -inputs = ["**/*2.13*/**/jacoco.xml"] -exclude_paths = [] -verbose = true - -rules = [ - # filter-out method from AtumAgent class - #"method:za.co.absa.atum.agent.AtumAgent#dispatcherFromConfig*", - - # filter-out method from AtumAgent object - #"method:za.co.absa.atum.agent.AtumAgent$#dispatcherFromConfig*" -] diff --git a/jmf-rules.txt b/jmf-rules.txt index 491d545a..e0857758 100644 --- a/jmf-rules.txt +++ b/jmf-rules.txt @@ -150,5 +150,29 @@ *#* synthetic name-contains:$anonfun$ id:scala-anonfun # ───────────────────────────────────────────────────────────────────────────── -# DIRECT RULES +# PROJECT RULES # ───────────────────────────────────────────────────────────────────────────── + +*.api.http.*#* +*.config.*#* +za.co.absa.atum.agent.dispatcher.Dispatcher#* +za.co.absa.atum.agent.dispatcher.ConsoleDispatcher#* +za.co.absa.atum.server.Main*#* +za.co.absa.atum.server.Constants*#* +za.co.absa.atum.server.api.database.DoobieImplicits*#* +za.co.absa.atum.server.api.database.TransactorProvider*#* +za.co.absa.atum.server.api.common.http.Routes*#* +za.co.absa.atum.server.api.v2.repository.PartitioningRepository#* +za.co.absa.atum.server.api.v2.repository.PartitioningRepository$#* +za.co.absa.atum.server.api.v2.controller.PartitioningController#* +za.co.absa.atum.server.api.v2.controller.PartitioningController$#* +za.co.absa.atum.server.api.v2.service.PartitioningService#* +za.co.absa.atum.server.api.v2.service.PartitioningService$#* +za.co.absa.atum.server.api.common.http.Routes*#* +za.co.absa.atum.server.implicits.SeqImplicits*#* +za.co.absa.atum.model.envelopes.Pagination#* +za.co.absa.atum.model.dto.PartitioningParentPatchDTO*#* +za.co.absa.atum.model.ApiPaths*#* +za.co.absa.atum.model.envelopes.ResponseEnvelope#* +za.co.absa.atum.model.envelopes.StatusResponse#* +za.co.absa.atum.model.envelopes.SuccessResponse#*" diff --git a/project/FilteredJacocoAgentPlugin.scala b/project/FilteredJacocoAgentPlugin.scala index b0f6f40d..0fd901bf 100644 --- a/project/FilteredJacocoAgentPlugin.scala +++ b/project/FilteredJacocoAgentPlugin.scala @@ -68,6 +68,8 @@ object FilteredJacocoAgentPlugin extends AutoPlugin { } } + lazy val Jmf = config("jmf").extend(Compile) + // ---- commands private lazy val jacocoCleanAllCmd = Command.command("jacocoCleanAll") { state => val targets = enabledUnder(state) @@ -75,10 +77,25 @@ object FilteredJacocoAgentPlugin extends AutoPlugin { else targets.foldLeft(state) { (st, ref) => Command.process(s"${ref.project}/jacocoClean", st) } } - private lazy val jacocoReportAllCmd = Command.command("jacocoReportAll") { state => - val targets = enabledUnder(state) - if (targets.isEmpty) { println("[jacoco] nothing to report (no enabled modules under this aggregate)."); state } - else targets.foldLeft(state) { (st, ref) => Command.process(s"${ref.project}/jacocoReport", st) } + private lazy val jacocoReportAllCmd = Command.command("jacocoReportAll") { state0 => + val e = Project.extract(state0) + val current = e.currentRef + // your existing helper (enabled projects under current aggregate) + val under = enabledUnder(state0) + + // Also include current project if enabled + val selfEnabled = + e.getOpt(current / jacocoPluginEnabled).getOrElse(false) + + val targets = (if (selfEnabled) current +: under else under).distinct + + if (targets.isEmpty) { + println("[jacoco] nothing to report (no enabled modules here)."); state0 + } else { + targets.foldLeft(state0) { (st, ref) => + Command.process(s"${ref.project}/jacocoReport", st) + } + } } // ---- global defaults so keys exist everywhere (safe no-ops on projects without the plugin) @@ -119,7 +136,7 @@ object FilteredJacocoAgentPlugin extends AutoPlugin { // pull the agent with the runtime classifier (this is the actual -javaagent jar) ("org.jacoco" % "org.jacoco.agent" % jacocoVersion.value % Test).classifier("runtime"), ("org.jacoco" % "org.jacoco.cli" % jacocoVersion.value % Test).classifier("nodeps"), - "io.github.moranaapps" % "jacoco-method-filter-core_2.12" % jmfCoreVersion.value % Jmf.name, + "io.github.moranaapps" %% "jacoco-method-filter-core" % jmfCoreVersion.value % Jmf.name, ), jacocoSetUserDirToBuildRoot := true, @@ -147,15 +164,36 @@ object FilteredJacocoAgentPlugin extends AutoPlugin { // the rewrite task (your code, lightly cleaned) jmfRewrite := { - val log = streams.value.log - val enabled = jacocoPluginEnabled.value - + // --- hoist all .value lookups BEFORE conditionals --- // ensure classes exist (safe to always do; test would compile anyway) val _ = (Compile / compile).value - val classesIn = (Compile / classDirectory).value val rules = jmfRulesFile.value val upd = (Jmf / update).value // hoisted + val log = streams.value.log + val outRoot = jmfOutDir.value + val mainCls = jmfCliMain.value + val dryRun = jmfDryRun.value + val workDir = baseDirectory.value + val classesIn = (Compile / classDirectory).value + val rulesFile = jmfRulesFile.value + val enabled = jacocoPluginEnabled.value + + // Compile classpath (scala-stdlib, scopt, your module classes, etc.) + val compileCp: Seq[File] = Attributed.data((Compile / fullClasspath).value) + + // Jmf-resolved jars (your jacoco-method-filter-core, etc.) + val jmfJars: Seq[File] = (Jmf / update).value.matching(artifactFilter(`type` = "jar")).distinct + + // Final runtime CP + val cp: Seq[File] = (compileCp ++ jmfJars :+ (Compile / classDirectory).value).distinct + val cpStr = cp.distinct.map(_.getAbsolutePath).mkString(java.io.File.pathSeparator) + + val javaBin = { + val h = sys.props.get("java.home").getOrElse("") + if (h.nonEmpty) new java.io.File(new java.io.File(h, "bin"), "java").getAbsolutePath else "java" + } + // ---------------------------------------------------- if (!enabled) classesIn else if (!classesIn.exists) { @@ -168,18 +206,17 @@ object FilteredJacocoAgentPlugin extends AutoPlugin { val outDir = jmfOutDir.value / "classes-filtered" IO.delete(outDir); IO.createDirectory(outDir) - val toolJars = upd.matching(artifactFilter(`type` = "jar")).distinct - log.info("[jmf] tool CP:\n" + toolJars.map(f => s" - ${f.getAbsolutePath}").mkString("\n")) + log.info("[jmf] runtime CP:\n" + cp.map(f => s" - ${f.getAbsolutePath}").mkString("\n")) - val cpStr = toolJars.mkString(java.io.File.pathSeparator) - val args = Seq("java","-cp", cpStr, jmfCliMain.value, - "--in", classesIn.getAbsolutePath, - "--out", outDir.getAbsolutePath, - "--rules", rules.getAbsolutePath) ++ - (if (jmfDryRun.value) Seq("--dry-run") else Seq()) + val args = Seq( + javaBin, "-cp", cpStr, jmfCliMain.value, + "--in", classesIn.getAbsolutePath, + "--out", outDir.getAbsolutePath, + "--rules", rules.getAbsolutePath + ) ++ (if (jmfDryRun.value) Seq("--dry-run") else Seq()) log.info(s"[jmf] rewrite: ${args.mkString(" ")}") - val code = scala.sys.process.Process(args, baseDirectory.value).! + val code = scala.sys.process.Process(args, workDir).! if (code != 0) sys.error(s"[jmf] rewriter failed ($code)") outDir } diff --git a/project/Setup.scala b/project/Setup.scala index 72057e8c..fa5df055 100644 --- a/project/Setup.scala +++ b/project/Setup.scala @@ -14,7 +14,6 @@ * limitations under the License. */ -//import com.github.sbt.jacoco.JacocoKeys.jacocoExcludes import sbt.* import sbt.Keys.* import sbtassembly.AssemblyKeys.assemblyMergeStrategy @@ -37,7 +36,6 @@ object Setup { lazy val commonSettings: Seq[SettingsDefinition] = Seq( scalacOptions ++= Setup.commonScalacOptions, Test / parallelExecution := false, -// jacocoExcludes := JacocoSetup.jacocoProjectExcludes(), (assembly / test) := {}, (publish / test) := { (Test / testOnly).toTask(" *UnitTests").value } ) diff --git a/project/VersionAxes.scala b/project/VersionAxes.scala index 9d3e319c..27e3dbd5 100644 --- a/project/VersionAxes.scala +++ b/project/VersionAxes.scala @@ -16,8 +16,6 @@ import sbt.{Def, ModuleID, VirtualAxis, *} import sbt.Keys.* import sbt.internal.ProjectMatrix -//import JacocoSetup.jacocoSettings -//import com.github.sbt.jacoco.JacocoKeys.jacocoReportSettings import za.co.absa.commons.version.Version @@ -55,7 +53,6 @@ object VersionAxes { libraryDependencies ++= dependenciesFnc(sparkVersion, scalaVersion), printVersionInfo := streams.value.log.info(s"Building ${name.value} with Spark $sparkVersion, Scala ${scalaVersion.asString}"), (Compile / compile) := ((Compile / compile) dependsOn printVersionInfo).value, -// jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion, name.value), ).settings(settings *) ) } @@ -73,7 +70,6 @@ object VersionAxes { libraryDependencies ++= dependenciesFnc(scalaVersion), printVersionInfo := streams.value.log.info(s"Building ${name.value} with Scala ${scalaVersion.asString}"), (Compile / compile) := ((Compile / compile) dependsOn printVersionInfo).value, -// jacocoReportSettings := jacocoSettings(scalaVersion, name.value), ).settings(settings *) ) ) @@ -90,7 +86,6 @@ object VersionAxes { scalacOptions ++= Setup.serverAndDbScalacOptions, printVersionInfo := streams.value.log.info(s"Building ${name.value} with Scala ${scalaVersion.asString}"), (Compile / compile) := ((Compile / compile) dependsOn printVersionInfo).value, -// jacocoReportSettings := jacocoSettings(scalaVersion, name.value), ).settings(settings *) ) } From 63964f4686bcdc9519f792d8e4661b0c16c2389c Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Wed, 3 Sep 2025 17:07:38 +0200 Subject: [PATCH 30/38] Revert to previous version as there is a bug. --- .github/workflows/jacoco_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 2af2cfb2..7fdbb5b4 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -77,7 +77,7 @@ jobs: - name: Check coverage thresholds and add reports in PR comments id: jacoco - uses: MoranaApps/jacoco-report@v2 + uses: MoranaApps/jacoco-report@v2.1.0 with: token: '${{ secrets.GITHUB_TOKEN }}' paths: | From 4c1660ce2b8a11e51a01efb69a74d22af08812c3 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Wed, 3 Sep 2025 17:12:08 +0200 Subject: [PATCH 31/38] Comment out project related jmf rules. --- .github/workflows/jacoco_report.yml | 2 +- jmf-rules.txt | 48 +++++++++++++++-------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 7fdbb5b4..2af2cfb2 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -77,7 +77,7 @@ jobs: - name: Check coverage thresholds and add reports in PR comments id: jacoco - uses: MoranaApps/jacoco-report@v2.1.0 + uses: MoranaApps/jacoco-report@v2 with: token: '${{ secrets.GITHUB_TOKEN }}' paths: | diff --git a/jmf-rules.txt b/jmf-rules.txt index e0857758..02a33ee9 100644 --- a/jmf-rules.txt +++ b/jmf-rules.txt @@ -153,26 +153,28 @@ # PROJECT RULES # ───────────────────────────────────────────────────────────────────────────── -*.api.http.*#* -*.config.*#* -za.co.absa.atum.agent.dispatcher.Dispatcher#* -za.co.absa.atum.agent.dispatcher.ConsoleDispatcher#* -za.co.absa.atum.server.Main*#* -za.co.absa.atum.server.Constants*#* -za.co.absa.atum.server.api.database.DoobieImplicits*#* -za.co.absa.atum.server.api.database.TransactorProvider*#* -za.co.absa.atum.server.api.common.http.Routes*#* -za.co.absa.atum.server.api.v2.repository.PartitioningRepository#* -za.co.absa.atum.server.api.v2.repository.PartitioningRepository$#* -za.co.absa.atum.server.api.v2.controller.PartitioningController#* -za.co.absa.atum.server.api.v2.controller.PartitioningController$#* -za.co.absa.atum.server.api.v2.service.PartitioningService#* -za.co.absa.atum.server.api.v2.service.PartitioningService$#* -za.co.absa.atum.server.api.common.http.Routes*#* -za.co.absa.atum.server.implicits.SeqImplicits*#* -za.co.absa.atum.model.envelopes.Pagination#* -za.co.absa.atum.model.dto.PartitioningParentPatchDTO*#* -za.co.absa.atum.model.ApiPaths*#* -za.co.absa.atum.model.envelopes.ResponseEnvelope#* -za.co.absa.atum.model.envelopes.StatusResponse#* -za.co.absa.atum.model.envelopes.SuccessResponse#*" +# Commented out to reconsider if needed + waiting for jacoco-report bug fix + +#*.api.http.*#* +#*.config.*#* +#za.co.absa.atum.agent.dispatcher.Dispatcher#* +#za.co.absa.atum.agent.dispatcher.ConsoleDispatcher#* +#za.co.absa.atum.server.Main*#* +#za.co.absa.atum.server.Constants*#* +#za.co.absa.atum.server.api.database.DoobieImplicits*#* +#za.co.absa.atum.server.api.database.TransactorProvider*#* +#za.co.absa.atum.server.api.common.http.Routes*#* +#za.co.absa.atum.server.api.v2.repository.PartitioningRepository#* +#za.co.absa.atum.server.api.v2.repository.PartitioningRepository$#* +#za.co.absa.atum.server.api.v2.controller.PartitioningController#* +#za.co.absa.atum.server.api.v2.controller.PartitioningController$#* +#za.co.absa.atum.server.api.v2.service.PartitioningService#* +#za.co.absa.atum.server.api.v2.service.PartitioningService$#* +#za.co.absa.atum.server.api.common.http.Routes*#* +#za.co.absa.atum.server.implicits.SeqImplicits*#* +#za.co.absa.atum.model.envelopes.Pagination#* +#za.co.absa.atum.model.dto.PartitioningParentPatchDTO*#* +#za.co.absa.atum.model.ApiPaths*#* +#za.co.absa.atum.model.envelopes.ResponseEnvelope#* +#za.co.absa.atum.model.envelopes.StatusResponse#* +#za.co.absa.atum.model.envelopes.SuccessResponse#*" From 3c7011e057d890123ce8ab78873d62e9dd92e43b Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Wed, 3 Sep 2025 17:16:44 +0200 Subject: [PATCH 32/38] Remove fake changes after testing. --- agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala | 3 --- .../scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala | 1 - reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala | 2 -- server/src/main/scala/za/co/absa/atum/server/Main.scala | 3 --- 4 files changed, 9 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index fc991d9d..458c57dc 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -46,9 +46,6 @@ class AtumContext private[agent] ( */ def currentMeasures: Set[AtumMeasure] = measures - - - /** * Returns the sub-partition context in the AtumContext. * diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala index 11c7ab60..17ec0398 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataDTO.scala @@ -26,5 +26,4 @@ case class AdditionalDataDTO( object AdditionalDataDTO { implicit val encodeAdditionalDataDTO: Encoder[AdditionalDataDTO] = deriveEncoder implicit val decodeAdditionalDataDTO: Decoder[AdditionalDataDTO] = deriveDecoder - } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 9c660f33..a07d95b8 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -62,8 +62,6 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit } yield checkpointsOrError } - - /** * Function to retrieve a page of checkpoints of the given name belonging to the flow. * The checkpoints are ordered by their creation order. diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 9a2ab939..cb159391 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -129,9 +129,6 @@ object Main extends ZIOAppDefault { } - - - override val bootstrap: ZLayer[Any, Config.Error, Unit] = Runtime.removeDefaultLoggers >>> SLF4J.slf4j >>> Runtime.setConfigProvider(configProvider) From 6e8c089eb7f5646ce6eca03c8199c224936515f5 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 4 Sep 2025 08:16:39 +0200 Subject: [PATCH 33/38] Applied latest jmf release. --- build.sbt | 2 +- jmf-rules.txt | 3 ++- project/FilteredJacocoAgentPlugin.scala | 4 +++- project/JacocoBaseKeysPlugin.scala | 2 ++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 1a5adda8..e9e697b5 100644 --- a/build.sbt +++ b/build.sbt @@ -143,5 +143,5 @@ lazy val reader = (projectMatrix in file("reader")) ): _* ) .addScalaCrossBuild(Setup.clientSupportedScalaVersions, Dependencies.readerDependencies) - .enablePlugins(FilteredJacocoAgentPlugin) + .enablePlugins(FilteredJacocoAgentPlugin) .dependsOn(model) diff --git a/jmf-rules.txt b/jmf-rules.txt index 02a33ee9..85b7ea54 100644 --- a/jmf-rules.txt +++ b/jmf-rules.txt @@ -1,4 +1,5 @@ -# jacoco-method-filter — Default Rules & HowTo (Scala + Java) +# jacoco-method-filter — Default Rules & HowTo (Scala) +# [jmf:1.0.0] # # This file defines which methods should be annotated as *Generated so JaCoCo ignores them. # One rule per line. diff --git a/project/FilteredJacocoAgentPlugin.scala b/project/FilteredJacocoAgentPlugin.scala index 0fd901bf..c02eadf0 100644 --- a/project/FilteredJacocoAgentPlugin.scala +++ b/project/FilteredJacocoAgentPlugin.scala @@ -1,3 +1,5 @@ +// JacocoBaseKeysPlugin.scala | last modified in v1.0.0 + import JacocoBaseKeysPlugin.autoImport.* import sbt.* import sbt.Keys.* @@ -131,7 +133,7 @@ object FilteredJacocoAgentPlugin extends AutoPlugin { // ---- coordinates jacocoVersion := "0.8.12", - jmfCoreVersion := "0.1.7", + jmfCoreVersion := "1.0.0", libraryDependencies ++= Seq( // pull the agent with the runtime classifier (this is the actual -javaagent jar) ("org.jacoco" % "org.jacoco.agent" % jacocoVersion.value % Test).classifier("runtime"), diff --git a/project/JacocoBaseKeysPlugin.scala b/project/JacocoBaseKeysPlugin.scala index cd2ab0a3..338342e1 100644 --- a/project/JacocoBaseKeysPlugin.scala +++ b/project/JacocoBaseKeysPlugin.scala @@ -1,3 +1,5 @@ +// JacocoBaseKeysPlugin.scala | last modified in v1.0.0 + import sbt.* import sbt.Keys.* From a029d4354c4a0680c2b7d4075bc0a8cc969824a8 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 4 Sep 2025 08:19:36 +0200 Subject: [PATCH 34/38] Expansion of jacoco pipeline. --- .github/workflows/jacoco_report.yml | 41 ++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 2af2cfb2..6eb44480 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -65,11 +65,50 @@ jobs: - name: Setup Scala uses: olafurpg/setup-scala@v14 with: - java-version: "adopt@1.11.0-11" + java-version: 'adopt@1.8' - name: Prepare testing database run: sbt flywayMigrate + - name: Build and run tests with test coverage + continue-on-error: true + id: jacocorun + run: sbt jacoco + + - name: Check coverage thresholds and add reports in PR comments + id: jacoco + uses: MoranaApps/jacoco-report@v2 + with: + token: '${{ secrets.GITHUB_TOKEN }}' + paths: | + **/target/**/jacoco/report/jacoco.xml + exclude-paths: | + database/target/** + sensitivity: "detail" + comment-mode: 'multi' + min-coverage-overall: ${{ env.coverage-overall }} + min-coverage-changed-files: ${{ env.coverage-changed-files }} + min-coverage-per-changed-file: ${{ env.coverage-per-changed-file }} + skip-unchanged: false + modules: ${{ env.MODULES }} + modules-thresholds: ${{ env.MODULES_THRESHOLDS }} + + + jacoco-report-server: + name: JaCoCo Report Server + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup Scala + uses: olafurpg/setup-scala@v14 + with: + java-version: "adopt@1.11.0-11" + - name: Build and run tests with test coverage continue-on-error: true id: jacocorun From cf6220d2b32901595680c39d1088582fae623959 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 4 Sep 2025 08:30:19 +0200 Subject: [PATCH 35/38] Re-setup of minimal values for Overall. --- .github/workflows/jacoco_report.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 6eb44480..dfdcb7f0 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -34,8 +34,10 @@ env: atum-reader: reader atum-server: server MODULES_THRESHOLDS: | - atum-model: 56*80* - atum-server: 66*80* + atum-reader: 74*80* + atum-agent: 67*80* + atum-model: 35*80* + atum-server: 71*80* jobs: jacoco-report: From 7a90be4449e28fbf66ddc7af1d5b0dda077014d6 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 4 Sep 2025 09:16:06 +0200 Subject: [PATCH 36/38] Return back db usage for server part. --- .github/workflows/jacoco_report.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index dfdcb7f0..3a3f4a29 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -100,6 +100,20 @@ jobs: name: JaCoCo Report Server runs-on: ubuntu-latest + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: atum_db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: Checkout code uses: actions/checkout@v4 @@ -111,6 +125,9 @@ jobs: with: java-version: "adopt@1.11.0-11" + - name: Prepare testing database + run: sbt flywayMigrate + - name: Build and run tests with test coverage continue-on-error: true id: jacocorun From 145c0bd599904fb5e80e7d22253c62322eb180e1 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Fri, 5 Sep 2025 07:41:58 +0200 Subject: [PATCH 37/38] Activation of rules and test of jacoco-report usage. --- .../dispatcher/CapturingDispatcher.scala | 3 ++ jmf-rules.txt | 46 +++++++++---------- .../atum/model/envelopes/StatusResponse.scala | 3 ++ .../za/co/absa/atum/reader/core/Reader.scala | 2 + .../scala/za/co/absa/atum/server/Main.scala | 3 ++ 5 files changed, 34 insertions(+), 23 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/CapturingDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/CapturingDispatcher.scala index 5201d992..2b03a43b 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/CapturingDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/CapturingDispatcher.scala @@ -33,6 +33,9 @@ class CapturingDispatcher(config: Config) extends Dispatcher(config) { val captureLimit: Int = config.getInt(CheckpointLimitKey) + + + /** * This method is used to clear all captured data. */ diff --git a/jmf-rules.txt b/jmf-rules.txt index 85b7ea54..af5e2755 100644 --- a/jmf-rules.txt +++ b/jmf-rules.txt @@ -156,26 +156,26 @@ # Commented out to reconsider if needed + waiting for jacoco-report bug fix -#*.api.http.*#* -#*.config.*#* -#za.co.absa.atum.agent.dispatcher.Dispatcher#* -#za.co.absa.atum.agent.dispatcher.ConsoleDispatcher#* -#za.co.absa.atum.server.Main*#* -#za.co.absa.atum.server.Constants*#* -#za.co.absa.atum.server.api.database.DoobieImplicits*#* -#za.co.absa.atum.server.api.database.TransactorProvider*#* -#za.co.absa.atum.server.api.common.http.Routes*#* -#za.co.absa.atum.server.api.v2.repository.PartitioningRepository#* -#za.co.absa.atum.server.api.v2.repository.PartitioningRepository$#* -#za.co.absa.atum.server.api.v2.controller.PartitioningController#* -#za.co.absa.atum.server.api.v2.controller.PartitioningController$#* -#za.co.absa.atum.server.api.v2.service.PartitioningService#* -#za.co.absa.atum.server.api.v2.service.PartitioningService$#* -#za.co.absa.atum.server.api.common.http.Routes*#* -#za.co.absa.atum.server.implicits.SeqImplicits*#* -#za.co.absa.atum.model.envelopes.Pagination#* -#za.co.absa.atum.model.dto.PartitioningParentPatchDTO*#* -#za.co.absa.atum.model.ApiPaths*#* -#za.co.absa.atum.model.envelopes.ResponseEnvelope#* -#za.co.absa.atum.model.envelopes.StatusResponse#* -#za.co.absa.atum.model.envelopes.SuccessResponse#*" +*.api.http.*#* +*.config.*#* +za.co.absa.atum.agent.dispatcher.Dispatcher#* +za.co.absa.atum.agent.dispatcher.ConsoleDispatcher#* +za.co.absa.atum.server.Main*#* +za.co.absa.atum.server.Constants*#* +za.co.absa.atum.server.api.database.DoobieImplicits*#* +za.co.absa.atum.server.api.database.TransactorProvider*#* +za.co.absa.atum.server.api.common.http.Routes*#* +za.co.absa.atum.server.api.v2.repository.PartitioningRepository#* +za.co.absa.atum.server.api.v2.repository.PartitioningRepository$#* +za.co.absa.atum.server.api.v2.controller.PartitioningController#* +za.co.absa.atum.server.api.v2.controller.PartitioningController$#* +za.co.absa.atum.server.api.v2.service.PartitioningService#* +za.co.absa.atum.server.api.v2.service.PartitioningService$#* +za.co.absa.atum.server.api.common.http.Routes*#* +za.co.absa.atum.server.implicits.SeqImplicits*#* +za.co.absa.atum.model.envelopes.Pagination#* +za.co.absa.atum.model.dto.PartitioningParentPatchDTO*#* +za.co.absa.atum.model.ApiPaths*#* +za.co.absa.atum.model.envelopes.ResponseEnvelope#* +za.co.absa.atum.model.envelopes.StatusResponse#* +za.co.absa.atum.model.envelopes.SuccessResponse#*" diff --git a/model/src/main/scala/za/co/absa/atum/model/envelopes/StatusResponse.scala b/model/src/main/scala/za/co/absa/atum/model/envelopes/StatusResponse.scala index 8f514124..5ec47e76 100644 --- a/model/src/main/scala/za/co/absa/atum/model/envelopes/StatusResponse.scala +++ b/model/src/main/scala/za/co/absa/atum/model/envelopes/StatusResponse.scala @@ -25,6 +25,9 @@ object StatusResponse { implicit val encoder: io.circe.Encoder[StatusResponse] = deriveEncoder implicit val decoder: io.circe.Decoder[StatusResponse] = deriveDecoder + + + lazy val up: StatusResponse = { StatusResponse( status = "UP", diff --git a/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala index 46b80ebd..94e7ba03 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala @@ -48,6 +48,8 @@ abstract class Reader[F[_]](implicit case Left(a) => me.unit(Left(a)) } + + protected def getQuery[R: Decoder]( endpointUri: String, params: Map[String, String] = Map.empty diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index cb159391..9a2ab939 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -129,6 +129,9 @@ object Main extends ZIOAppDefault { } + + + override val bootstrap: ZLayer[Any, Config.Error, Unit] = Runtime.removeDefaultLoggers >>> SLF4J.slf4j >>> Runtime.setConfigProvider(configProvider) From dcffa6190e571f967f2e8c838f4950b0c390732c Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Fri, 5 Sep 2025 07:49:22 +0200 Subject: [PATCH 38/38] Removed from test changes. --- .../za/co/absa/atum/agent/dispatcher/CapturingDispatcher.scala | 3 --- .../scala/za/co/absa/atum/model/envelopes/StatusResponse.scala | 3 --- reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala | 2 -- server/src/main/scala/za/co/absa/atum/server/Main.scala | 3 --- 4 files changed, 11 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/CapturingDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/CapturingDispatcher.scala index 2b03a43b..5201d992 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/CapturingDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/CapturingDispatcher.scala @@ -33,9 +33,6 @@ class CapturingDispatcher(config: Config) extends Dispatcher(config) { val captureLimit: Int = config.getInt(CheckpointLimitKey) - - - /** * This method is used to clear all captured data. */ diff --git a/model/src/main/scala/za/co/absa/atum/model/envelopes/StatusResponse.scala b/model/src/main/scala/za/co/absa/atum/model/envelopes/StatusResponse.scala index 5ec47e76..8f514124 100644 --- a/model/src/main/scala/za/co/absa/atum/model/envelopes/StatusResponse.scala +++ b/model/src/main/scala/za/co/absa/atum/model/envelopes/StatusResponse.scala @@ -25,9 +25,6 @@ object StatusResponse { implicit val encoder: io.circe.Encoder[StatusResponse] = deriveEncoder implicit val decoder: io.circe.Decoder[StatusResponse] = deriveDecoder - - - lazy val up: StatusResponse = { StatusResponse( status = "UP", diff --git a/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala index 94e7ba03..46b80ebd 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala @@ -48,8 +48,6 @@ abstract class Reader[F[_]](implicit case Left(a) => me.unit(Left(a)) } - - protected def getQuery[R: Decoder]( endpointUri: String, params: Map[String, String] = Map.empty diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 9a2ab939..cb159391 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -129,9 +129,6 @@ object Main extends ZIOAppDefault { } - - - override val bootstrap: ZLayer[Any, Config.Error, Unit] = Runtime.removeDefaultLoggers >>> SLF4J.slf4j >>> Runtime.setConfigProvider(configProvider)