From 836971dc9afee340d7c94bca283f675bf4530f1d Mon Sep 17 00:00:00 2001
From: Jan-Niclas Struewer <j.n.struewer@gmail.com>
Date: Tue, 23 May 2023 21:23:24 +0200
Subject: [PATCH] updated db structure to enable a query to get the latest
results for all tools for a certain repository
---
.../iem/dataprovider/gitlab/Repository.kt | 6 +-
.../iem/dataprovider/sarif/SarifExtensions.kt | 20 ++----
.../dataprovider/taskManager/TaskManager.kt | 13 ++--
.../taskManager/tasks/DetektTask.kt | 9 +--
.../dataprovider/taskManager/tasks/OdcTask.kt | 8 +--
.../taskManager/tasks/ProcessTask.kt | 23 +++----
.../dataprovider/toolResult/RuleRepository.kt | 10 +++
.../dataprovider/toolResult/ToolRepository.kt | 12 ++++
.../iem/dataprovider/toolResult/ToolRun.kt | 46 +++++++++++--
.../dataprovider/toolResult/ToolRunService.kt | 67 +++++++++++++++++++
10 files changed, 163 insertions(+), 51 deletions(-)
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/RuleRepository.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRepository.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunService.kt
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
index fb409de3..9a6ccf02 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
@@ -14,7 +14,11 @@ class Repository {
var id: UUID? = null
@OrderBy("time_stamp DESC")
- @OneToMany(mappedBy = "repository", cascade = [CascadeType.ALL], orphanRemoval = true)
+ @OneToMany(
+ mappedBy = "repository",
+ cascade = [CascadeType.MERGE, CascadeType.REFRESH, CascadeType.REMOVE, CascadeType.DETACH],
+ orphanRemoval = true
+ )
var toolRuns: MutableList<ToolRun> = mutableListOf()
@Column(name = "name")
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
index b535c2e3..145f0cad 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
@@ -21,7 +21,6 @@ fun de.fraunhofer.iem.dataprovider.sarif.Rule.asDbObject(): Rule {
fun de.fraunhofer.iem.dataprovider.sarif.Tool.asDbObject(): Tool {
val tool = Tool()
- tool.addRules(this.driver.rules.map { it.asDbObject() })
tool.fullName = this.driver.fullName
tool.name = this.driver.name
tool.version = this.driver.version
@@ -38,7 +37,7 @@ fun Location.asDbObject(): ResultLocation {
return resultLocation
}
-fun Result.asDbObject(rules: Collection<Rule>): ToolResult {
+fun Result.asDbObject(): ToolResult {
val toolResult = ToolResult()
when (this.level.uppercase(Locale.getDefault())) {
@@ -48,26 +47,19 @@ fun Result.asDbObject(rules: Collection<Rule>): ToolResult {
else -> toolResult.level = Level.None
}
- rules
- .filter { it.sarifRuleId == this.ruleId }
- .forEach { it.addResultLocation(toolResult) }
-
toolResult.message = this.message.text
toolResult.addLocations(this.locations.map { it.asDbObject() })
return toolResult
}
-fun Sarif.asDbObject(): List<ToolRun> {
- return this.runs.map { run ->
- val tr = ToolRun()
- tr.tool = run.tool.asDbObject()
- tr.addToolResults(run.results.map { it.asDbObject(tr.tool!!.rules) })
- tr.timeStamp = Timestamp.from(Instant.now())
- tr
- }
+fun Run.asDbObject(): ToolRun {
+ val tr = ToolRun()
+ tr.timeStamp = Timestamp.from(Instant.now())
+ return tr
}
+
fun getSarifFromFilePath(resultPath: Path): Sarif {
val resFile = resultPath.toFile()
if (resFile.exists()) {
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
index d2fda8fa..33fab379 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
@@ -5,7 +5,7 @@ import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.sarif.Sarif
import de.fraunhofer.iem.dataprovider.taskManager.tasks.*
-import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunService
import jakarta.annotation.PreDestroy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -42,7 +42,7 @@ class DetektDone(override val taskId: UUID, val sarif: Sarif) : TaskDone()
class TaskManager(
private val config: Config,
private val repositoryRepository: RepositoryRepository,
- private val toolRunRepository: ToolRunRepository
+ private val toolRunService: ToolRunService
) {
// The used default dispatcher is ok for CPU-bound workloads. However,
@@ -128,8 +128,8 @@ class TaskManager(
analysisResultsPath,
::addEvent,
event.repoId,
- repositoryRepository,
- toolRunRepository, groupId
+ groupId,
+ toolRunService
)
val detektTask = DetektTask(
@@ -137,9 +137,8 @@ class TaskManager(
analysisResultsPath,
::addEvent,
event.repoId,
- repositoryRepository,
- toolRunRepository,
- groupID = groupId
+ groupID = groupId,
+ toolRunService
)
taskIds.add(odcTask.taskID)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
index af6ec9ae..40355b69 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
@@ -1,9 +1,7 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
-
-import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.taskManager.Event
-import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunService
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.Resource
import java.nio.file.Path
@@ -15,9 +13,8 @@ class DetektTask(
projectPath: String, outputPath: String,
override val responseChannel: suspend (task: Event) -> Unit,
override val repoId: UUID,
- override val repository: RepositoryRepository,
- override val toolRunRepository: ToolRunRepository,
- override val groupID: UUID? = null
+ override val groupID: UUID? = null,
+ override val toolRunService: ToolRunService
) : SarifTask() {
private val resource: Resource = ClassPathResource("scripts/detekt.sh")
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/OdcTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/OdcTask.kt
index 12c653c5..bfb92400 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/OdcTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/OdcTask.kt
@@ -1,8 +1,7 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
-import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.taskManager.Event
-import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunService
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.Resource
import java.nio.file.Path
@@ -15,9 +14,8 @@ class OdcTask(
projectPath: String, outputPath: String,
override val responseChannel: suspend (task: Event) -> Unit,
override val repoId: UUID,
- override val repository: RepositoryRepository,
- override val toolRunRepository: ToolRunRepository,
- override val groupID: UUID? = null
+ override val groupID: UUID? = null,
+ override val toolRunService: ToolRunService
) : SarifTask() {
private val resource: Resource = ClassPathResource("scripts/odc.sh")
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
index f1793de1..552f62c0 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
@@ -1,15 +1,14 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
-import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.sarif.Sarif
-import de.fraunhofer.iem.dataprovider.sarif.asDbObject
import de.fraunhofer.iem.dataprovider.sarif.getSarifFromFilePath
import de.fraunhofer.iem.dataprovider.taskManager.SarifProcessDone
import de.fraunhofer.iem.dataprovider.taskManager.SarifProcessGroupDone
-import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunService
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
+import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
@@ -48,8 +47,7 @@ sealed class ProcessTask : Task() {
sealed class SarifTask : ProcessTask() {
protected abstract val repoId: UUID
- protected abstract val repository: RepositoryRepository
- protected abstract val toolRunRepository: ToolRunRepository
+ protected abstract val toolRunService: ToolRunService
protected abstract val resultPath: Path
@@ -59,16 +57,13 @@ sealed class SarifTask : ProcessTask() {
// has findings it might return an exit code != 0 even tho the process finished correctly
logger.info("Handle Process return in $javaClass")
- val sarifResult = getSarifFromFilePath(resultPath)
- val repo = repository.findById(repoId)
- val sarifDb = sarifResult.asDbObject()
- val results = toolRunRepository.saveAll(sarifDb)
-
- repo.ifPresent {
- it.addToolResults(results)
- repository.save(it)
+ try {
+ val sarifResult = getSarifFromFilePath(resultPath)
+ toolRunService.saveSarif(sarifResult, repoId)
+ sendResult(sarifResult)
+ } catch (e: IOException) {
+ // TODO: send meaningful event and react on error
}
- sendResult(sarifResult)
}
override suspend fun sendResult(sarif: Sarif) {
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/RuleRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/RuleRepository.kt
new file mode 100644
index 00000000..ea3d707b
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/RuleRepository.kt
@@ -0,0 +1,10 @@
+package de.fraunhofer.iem.dataprovider.toolResult
+
+import org.springframework.data.jpa.repository.JpaRepository
+import java.util.*
+
+interface RuleRepository : JpaRepository<Rule, UUID> {
+
+
+ fun findByTool_Id(id: UUID): List<Rule>
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRepository.kt
new file mode 100644
index 00000000..1989e6ac
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRepository.kt
@@ -0,0 +1,12 @@
+package de.fraunhofer.iem.dataprovider.toolResult
+
+import org.springframework.data.jpa.repository.JpaRepository
+import java.util.*
+
+interface ToolRepository : JpaRepository<Tool, UUID> {
+ fun findByFullNameIgnoreCaseAndNameIgnoreCaseAndVersionIgnoreCase(
+ fullName: String?,
+ name: String?,
+ version: String?
+ ): Tool?
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
index 7b947239..76f05694 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
@@ -24,8 +24,7 @@ class ToolRun {
@Column(name = "id", nullable = false)
var id: UUID? = null
-
- @OneToOne(orphanRemoval = true, cascade = [CascadeType.ALL])
+ @OneToOne(orphanRemoval = true, cascade = [CascadeType.REFRESH, CascadeType.REMOVE, CascadeType.DETACH])
@JoinColumn(name = "tool_id")
var tool: Tool? = null
@@ -36,6 +35,24 @@ class ToolRun {
@ManyToOne(fetch = FetchType.EAGER)
var repository: Repository? = null
+ @OneToMany(
+ mappedBy = "toolRun",
+ cascade = [CascadeType.MERGE, CascadeType.REFRESH, CascadeType.REMOVE, CascadeType.DETACH],
+ orphanRemoval = true
+ )
+ var rules: MutableSet<Rule> = mutableSetOf()
+
+ fun addRules(rules: Collection<Rule>) {
+ rules.forEach {
+ this.addRule(it)
+ }
+ }
+
+ fun addRule(rule: Rule) {
+ rules.add(rule)
+ rule.toolRun = this
+ }
+
@OneToMany(mappedBy = "toolRun", cascade = [CascadeType.ALL], orphanRemoval = true)
var toolResults: MutableSet<ToolResult> = mutableSetOf()
@@ -68,7 +85,11 @@ class Tool {
@Column(name = "version")
var version: String? = null
- @OneToMany(mappedBy = "tool", cascade = [CascadeType.ALL], orphanRemoval = true)
+ @OneToMany(
+ mappedBy = "tool",
+ cascade = [CascadeType.MERGE, CascadeType.REFRESH, CascadeType.REMOVE, CascadeType.DETACH],
+ orphanRemoval = true
+ )
var rules: MutableSet<Rule> = mutableSetOf()
fun addRules(rules: Collection<Rule>) {
@@ -81,6 +102,16 @@ class Tool {
rules.add(rule)
rule.tool = this
}
+
+ override fun equals(other: Any?): Boolean {
+ if (this === other) return true
+ if (other == null || Hibernate.getClass(this) != Hibernate.getClass(other)) return false
+ other as Tool
+
+ return id != null && id == other.id
+ }
+
+ override fun hashCode(): Int = javaClass.hashCode()
}
@@ -186,9 +217,16 @@ class Rule {
@Column(name = "short_description", columnDefinition = "TEXT")
var shortDescription: String? = null
- @OneToMany(mappedBy = "rule", cascade = [CascadeType.ALL], orphanRemoval = true)
+ @OneToMany(
+ mappedBy = "rule",
+ cascade = [CascadeType.MERGE, CascadeType.REFRESH, CascadeType.REFRESH, CascadeType.DETACH],
+ orphanRemoval = true
+ )
var toolResults: MutableSet<ToolResult> = mutableSetOf()
+ @ManyToOne
+ var toolRun: ToolRun? = null
+
fun addResultLocations(toolResults: Collection<ToolResult>) {
toolResults.forEach {
this.addResultLocation(it)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunService.kt
new file mode 100644
index 00000000..36f8edc1
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunService.kt
@@ -0,0 +1,67 @@
+package de.fraunhofer.iem.dataprovider.toolResult
+
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
+import de.fraunhofer.iem.dataprovider.sarif.Sarif
+import de.fraunhofer.iem.dataprovider.sarif.asDbObject
+import org.springframework.stereotype.Service
+import java.util.*
+
+@Service
+class ToolRunService(
+ private val repositoryRepository: RepositoryRepository,
+ private val toolRunRepository: ToolRunRepository,
+ private val toolRepository: ToolRepository,
+ private val ruleRepository: RuleRepository
+) {
+
+ // TODO: this func should be a suspend fun. We can achieve this by
+ // wrapping the db return types in flux, mono, or futures
+ fun saveSarif(sarif: Sarif, repoId: UUID) {
+
+ val repo = repositoryRepository.findById(repoId)
+
+ sarif.runs.forEach { run ->
+ val fullName = run.tool.driver.fullName
+ val name = run.tool.driver.name
+ val version = run.tool.driver.version
+
+ val tool =
+ toolRepository.findByFullNameIgnoreCaseAndNameIgnoreCaseAndVersionIgnoreCase(fullName, name, version)
+ ?: toolRepository.save(run.tool.asDbObject())
+
+ val rules = if (tool.id != null) {
+ ruleRepository.findByTool_Id(tool.id!!).toMutableList()
+ } else {
+ mutableListOf()
+ }
+
+ run.tool.driver.rules.forEach { sarifRule ->
+ if (rules.find { it.sarifRuleId == sarifRule.id } == null) {
+ val rule = ruleRepository.save(sarifRule.asDbObject())
+ rules.add(rule)
+ tool.addRule(rule)
+ }
+ }
+
+ val toolResults = run.results.map { sarifResult ->
+ val tr = sarifResult.asDbObject()
+ rules.filter { it.sarifRuleId == sarifResult.ruleId }
+ .forEach { it.addResultLocation(tr) }
+ tr
+ }
+
+
+ val toolRun = run.asDbObject()
+ toolRun.tool = tool
+ toolRun.addRules(rules)
+ toolRun.addToolResults(toolResults)
+
+ val result = toolRunRepository.save(toolRun)
+ toolRepository.save(tool)
+ repo.ifPresent {
+ it.addToolResult(result)
+ repositoryRepository.save(it)
+ }
+ }
+ }
+}
\ No newline at end of file
--
GitLab