From 4854646b7a90cee94363e0f239a30b361351347b Mon Sep 17 00:00:00 2001
From: Jan-Niclas Struewer <j.n.struewer@gmail.com>
Date: Tue, 23 May 2023 22:45:12 +0200
Subject: [PATCH] code cleanup and db extension to support easy query for
latest tool results for each repository
---
.../iem/dataprovider/gitlab/Repository.kt | 9 +++
.../gitlab/RepositoryRepository.kt | 9 +--
.../iem/dataprovider/sarif/Sarif.kt | 44 ++++++------
.../iem/dataprovider/sarif/SarifExtensions.kt | 42 ++++++-----
.../dataprovider/taskManager/TaskManager.kt | 3 +-
.../taskManager/tasks/MetricsTask.kt | 13 ++--
.../iem/dataprovider/toolResult/ToolRun.kt | 69 +++++++++----------
.../toolResult/ToolRunRepository.kt | 5 +-
.../dataprovider/toolResult/ToolRunService.kt | 28 ++++++--
9 files changed, 120 insertions(+), 102 deletions(-)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
index 9a6ccf02..271ce238 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
@@ -1,5 +1,6 @@
package de.fraunhofer.iem.dataprovider.gitlab
+import de.fraunhofer.iem.dataprovider.toolResult.Tool
import de.fraunhofer.iem.dataprovider.toolResult.ToolRun
import jakarta.persistence.*
import java.util.*
@@ -34,6 +35,14 @@ class Repository {
@Column(name = "url")
var url: String? = null
+ @ManyToMany
+ @JoinTable(
+ name = "repository_tools",
+ joinColumns = [JoinColumn(name = "repository_id")],
+ inverseJoinColumns = [JoinColumn(name = "tools_id")]
+ )
+ var tools: MutableSet<Tool> = mutableSetOf()
+
fun addToolResults(toolRuns: Collection<ToolRun>) {
toolRuns.forEach {
this.addToolResult(it)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt
index 23fa8211..41476508 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt
@@ -1,13 +1,8 @@
-package de.fraunhofer.iem.dataprovider.gitlab;
+package de.fraunhofer.iem.dataprovider.gitlab
import org.springframework.data.jpa.repository.JpaRepository
import java.util.*
interface RepositoryRepository : JpaRepository<Repository, UUID> {
-
-
- fun existsByRepoIdAndPlatform(repoId: Long, platform: Platform): Boolean
-
-
fun findByRepoIdAndPlatform(repoId: Long, platform: Platform): Repository?
-}
\ No newline at end of file
+}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/Sarif.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/Sarif.kt
index 7768af14..17c97b21 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/Sarif.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/Sarif.kt
@@ -7,31 +7,31 @@ import kotlinx.serialization.Serializable
@Serializable
data class Sarif(
@SerialName("runs")
- val runs: List<Run>
+ val runs: List<SarifRun>
)
@Serializable
-data class Location(
+data class SarifLocation(
@SerialName("physicalLocation")
- val physicalLocation: PhysicalLocation
+ val physicalLocation: SarifPhysicalLocation
)
@Serializable
-data class Message(
+data class SarifMessage(
@SerialName("text")
val text: String
)
@Serializable
-data class PhysicalLocation(
+data class SarifPhysicalLocation(
@SerialName("artifactLocation")
- val artifactLocation: ArtifactLocation,
+ val artifactLocation: SarifArtifactLocation,
@SerialName("region")
- val region: Region
+ val region: SarifRegion
)
@Serializable
-data class Region(
+data class SarifRegion(
@SerialName("endColumn")
val endColumn: Int,
@SerialName("endLine")
@@ -43,19 +43,19 @@ data class Region(
)
@Serializable
-data class Result(
+data class SarifResult(
@SerialName("level")
val level: String,
@SerialName("locations")
- val locations: List<Location>,
+ val locations: List<SarifLocation>,
@SerialName("message")
- val message: Message,
+ val message: SarifMessage,
@SerialName("ruleId")
val ruleId: String
)
@Serializable
-data class Rule(
+data class SarifRule(
@SerialName("helpUri")
val helpUri: String,
@SerialName("id")
@@ -63,31 +63,31 @@ data class Rule(
@SerialName("name")
val name: String,
@SerialName("shortDescription")
- val shortDescription: ShortDescription
+ val shortDescription: SarifShortDescription
)
@Serializable
-data class Run(
+data class SarifRun(
@SerialName("results")
- val results: List<Result>,
+ val results: List<SarifResult>,
@SerialName("tool")
- val tool: Tool
+ val tool: SarifTool
)
@Serializable
-data class ShortDescription(
+data class SarifShortDescription(
@SerialName("text")
val text: String
)
@Serializable
-data class Tool(
+data class SarifTool(
@SerialName("driver")
- val driver: Driver
+ val driver: SarifDriver
)
@Serializable
-data class Driver(
+data class SarifDriver(
@SerialName("downloadUri")
val downloadUri: String? = null,
@SerialName("fullName")
@@ -103,7 +103,7 @@ data class Driver(
@SerialName("organization")
val organization: String? = null,
@SerialName("rules")
- val rules: List<Rule>,
+ val rules: List<SarifRule>,
@SerialName("semanticVersion")
val semanticVersion: String? = null,
@SerialName("version")
@@ -111,7 +111,7 @@ data class Driver(
)
@Serializable
-data class ArtifactLocation(
+data class SarifArtifactLocation(
@SerialName("uri")
val uri: String
)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
index 145f0cad..801d8f10 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
@@ -1,8 +1,6 @@
package de.fraunhofer.iem.dataprovider.sarif
import de.fraunhofer.iem.dataprovider.toolResult.*
-import de.fraunhofer.iem.dataprovider.toolResult.Rule
-import de.fraunhofer.iem.dataprovider.toolResult.Tool
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import java.io.IOException
@@ -11,7 +9,7 @@ import java.sql.Timestamp
import java.time.Instant
import java.util.*
-fun de.fraunhofer.iem.dataprovider.sarif.Rule.asDbObject(): Rule {
+fun SarifRule.asDbObject(): Rule {
val rule = Rule()
rule.sarifRuleId = this.id
rule.shortDescription = this.shortDescription.text
@@ -19,7 +17,7 @@ fun de.fraunhofer.iem.dataprovider.sarif.Rule.asDbObject(): Rule {
}
-fun de.fraunhofer.iem.dataprovider.sarif.Tool.asDbObject(): Tool {
+fun SarifTool.asDbObject(): Tool {
val tool = Tool()
tool.fullName = this.driver.fullName
tool.name = this.driver.name
@@ -27,33 +25,33 @@ fun de.fraunhofer.iem.dataprovider.sarif.Tool.asDbObject(): Tool {
return tool
}
-fun Location.asDbObject(): ResultLocation {
- val resultLocation = ResultLocation()
- resultLocation.endColumn = this.physicalLocation.region.endColumn
- resultLocation.endLine = this.physicalLocation.region.endLine
- resultLocation.startColumn = this.physicalLocation.region.startColumn
- resultLocation.startLine = this.physicalLocation.region.startLine
- resultLocation.artifactLocation = this.physicalLocation.artifactLocation.uri
- return resultLocation
+fun SarifLocation.asDbObject(): Location {
+ val location = Location()
+ location.endColumn = this.physicalLocation.region.endColumn
+ location.endLine = this.physicalLocation.region.endLine
+ location.startColumn = this.physicalLocation.region.startColumn
+ location.startLine = this.physicalLocation.region.startLine
+ location.artifactLocation = this.physicalLocation.artifactLocation.uri
+ return location
}
-fun Result.asDbObject(): ToolResult {
- val toolResult = ToolResult()
+fun SarifResult.asDbObject(): Finding {
+ val finding = Finding()
when (this.level.uppercase(Locale.getDefault())) {
- "WARNING" -> toolResult.level = Level.Warning
- "ERROR" -> toolResult.level = Level.Error
- "NOTE" -> toolResult.level = Level.Note
- else -> toolResult.level = Level.None
+ "WARNING" -> finding.level = Level.Warning
+ "ERROR" -> finding.level = Level.Error
+ "NOTE" -> finding.level = Level.Note
+ else -> finding.level = Level.None
}
- toolResult.message = this.message.text
- toolResult.addLocations(this.locations.map { it.asDbObject() })
+ finding.message = this.message.text
+ finding.addLocations(this.locations.map { it.asDbObject() })
- return toolResult
+ return finding
}
-fun Run.asDbObject(): ToolRun {
+fun SarifRun.asDbObject(): ToolRun {
val tr = ToolRun()
tr.timeStamp = Timestamp.from(Instant.now())
return tr
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
index 33fab379..324e5bab 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
@@ -31,7 +31,6 @@ class SarifProcessDone(override val taskId: UUID, val repoId: UUID, val sarif: S
class SarifProcessGroupDone(override val taskId: UUID, val repoId: UUID, val groupId: UUID, val sarif: Sarif) :
TaskDone()
-class DetektDone(override val taskId: UUID, val sarif: Sarif) : TaskDone()
/**
* The Task Manager takes tasks and distributes them to
@@ -157,7 +156,7 @@ class TaskManager(
eventGroup.remove(event.taskId)
if (eventGroup.isEmpty()) {
// TODO: Start metrics calculation
- worker.addTask(MetricsTask(event.repoId, repositoryRepository, ::addEvent))
+ worker.addTask(MetricsTask(event.repoId, toolRunService, ::addEvent))
}
}
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/MetricsTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/MetricsTask.kt
index 1d07ca14..a4d09731 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/MetricsTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/MetricsTask.kt
@@ -1,18 +1,17 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
-import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.taskManager.Event
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunService
import java.util.*
class MetricsTask(
val repoId: UUID,
- val repository: RepositoryRepository,
+ private val toolRunService: ToolRunService,
+
override val responseChannel: suspend (task: Event) -> Unit
) : Task() {
override suspend fun execute() {
-// val repo = repository.findById(repoId).get()
-// repo.toolRuns.forEach {tr ->
-//
-// }
+ logger.info("Starting metrics task for repoId $repoId")
+ toolRunService.getLatestToolRunsForRepo(repoId)
}
-}
\ No newline at end of file
+}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
index 76f05694..24c9c0ec 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
@@ -24,7 +24,7 @@ class ToolRun {
@Column(name = "id", nullable = false)
var id: UUID? = null
- @OneToOne(orphanRemoval = true, cascade = [CascadeType.REFRESH, CascadeType.REMOVE, CascadeType.DETACH])
+ @OneToOne(orphanRemoval = true)
@JoinColumn(name = "tool_id")
var tool: Tool? = null
@@ -37,11 +37,24 @@ class ToolRun {
@OneToMany(
mappedBy = "toolRun",
- cascade = [CascadeType.MERGE, CascadeType.REFRESH, CascadeType.REMOVE, CascadeType.DETACH],
orphanRemoval = true
)
var rules: MutableSet<Rule> = mutableSetOf()
+ @OneToMany(mappedBy = "toolRun", cascade = [CascadeType.ALL], orphanRemoval = true)
+ var findings: MutableSet<Finding> = mutableSetOf()
+
+ fun addFindings(findings: Collection<Finding>) {
+ findings.forEach {
+ this.addFinding(it)
+ }
+ }
+
+ fun addFinding(finding: Finding) {
+ findings.add(finding)
+ finding.toolRun = this
+ }
+
fun addRules(rules: Collection<Rule>) {
rules.forEach {
this.addRule(it)
@@ -52,20 +65,6 @@ class ToolRun {
rules.add(rule)
rule.toolRun = this
}
-
- @OneToMany(mappedBy = "toolRun", cascade = [CascadeType.ALL], orphanRemoval = true)
- var toolResults: MutableSet<ToolResult> = mutableSetOf()
-
- fun addToolResults(toolResults: Collection<ToolResult>) {
- toolResults.forEach {
- this.addToolResult(it)
- }
- }
-
- fun addToolResult(toolResult: ToolResult) {
- toolResults.add(toolResult)
- toolResult.toolRun = this
- }
}
@Entity
@@ -116,8 +115,8 @@ class Tool {
@Entity
-@Table(name = "tool_result")
-class ToolResult {
+@Table(name = "finding")
+class Finding {
@Id
@GeneratedValue(strategy = GenerationType.AUTO)
@Column(name = "id", nullable = false)
@@ -126,18 +125,18 @@ class ToolResult {
@ManyToOne
var rule: Rule? = null
- @OneToMany(mappedBy = "toolResult", cascade = [CascadeType.ALL], orphanRemoval = true)
- var resultLocations: MutableSet<ResultLocation> = mutableSetOf()
+ @OneToMany(mappedBy = "finding", cascade = [CascadeType.ALL], orphanRemoval = true)
+ var locations: MutableSet<Location> = mutableSetOf()
- fun addLocations(locations: Collection<ResultLocation>) {
+ fun addLocations(locations: Collection<Location>) {
locations.forEach {
this.addLocation(it)
}
}
- fun addLocation(location: ResultLocation) {
- resultLocations.add(location)
- location.toolResult = this
+ fun addLocation(location: Location) {
+ locations.add(location)
+ location.finding = this
}
@Enumerated
@@ -157,7 +156,7 @@ class ToolResult {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || Hibernate.getClass(this) != Hibernate.getClass(other)) return false
- other as ToolResult
+ other as Finding
return id != null && id == other.id
}
@@ -166,15 +165,15 @@ class ToolResult {
}
@Entity
-@Table(name = "result_location")
-class ResultLocation {
+@Table(name = "location")
+class Location {
@Id
@GeneratedValue(strategy = GenerationType.AUTO)
@Column(name = "id", nullable = false)
var id: UUID? = null
@ManyToOne
- var toolResult: ToolResult? = null
+ var finding: Finding? = null
@Column(name = "artifact_location", columnDefinition = "TEXT")
var artifactLocation: String? = null
@@ -194,7 +193,7 @@ class ResultLocation {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || Hibernate.getClass(this) != Hibernate.getClass(other)) return false
- other as ResultLocation
+ other as Location
return id != null && id == other.id
}
@@ -222,20 +221,20 @@ class Rule {
cascade = [CascadeType.MERGE, CascadeType.REFRESH, CascadeType.REFRESH, CascadeType.DETACH],
orphanRemoval = true
)
- var toolResults: MutableSet<ToolResult> = mutableSetOf()
+ var findings: MutableSet<Finding> = mutableSetOf()
@ManyToOne
var toolRun: ToolRun? = null
- fun addResultLocations(toolResults: Collection<ToolResult>) {
- toolResults.forEach {
+ fun addResultLocations(findings: Collection<Finding>) {
+ findings.forEach {
this.addResultLocation(it)
}
}
- fun addResultLocation(toolResult: ToolResult) {
- toolResults.add(toolResult)
- toolResult.rule = this
+ fun addResultLocation(finding: Finding) {
+ findings.add(finding)
+ finding.rule = this
}
@ElementCollection
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunRepository.kt
index 4a907621..a848eeef 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunRepository.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunRepository.kt
@@ -3,4 +3,7 @@ package de.fraunhofer.iem.dataprovider.toolResult
import org.springframework.data.jpa.repository.JpaRepository
import java.util.*
-interface ToolRunRepository : JpaRepository<ToolRun, UUID>
\ No newline at end of file
+interface ToolRunRepository : JpaRepository<ToolRun, UUID> {
+
+ fun findFirstByToolIdAndRepositoryIdOrderByTimeStampDesc(toolId: UUID, repoId: UUID): ToolRun
+}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunService.kt
index b537b1c0..03edbcf5 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRunService.kt
@@ -2,7 +2,7 @@ package de.fraunhofer.iem.dataprovider.toolResult
import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.sarif.Sarif
-import de.fraunhofer.iem.dataprovider.sarif.Tool
+import de.fraunhofer.iem.dataprovider.sarif.SarifTool
import de.fraunhofer.iem.dataprovider.sarif.asDbObject
import org.springframework.stereotype.Service
import java.util.*
@@ -15,6 +15,19 @@ class ToolRunService(
private val ruleRepository: RuleRepository
) {
+ fun getLatestToolRunsForRepo(repoId: UUID) {
+ repositoryRepository.findById(repoId).ifPresent { repo ->
+ val latestToolRuns = repo.tools.map { tool ->
+ toolRunRepository.findFirstByToolIdAndRepositoryIdOrderByTimeStampDesc(tool.id!!, repoId)
+ }
+ latestToolRuns.forEach { tr ->
+ println("TR Time: ${tr.timeStamp}")
+ }
+ // TODO: here we can calculate the metrics
+ }
+
+ }
+
// TODO: this func should be a suspend fun. We can achieve this by
// wrapping the db return types in flux, mono, or futures
fun saveSarif(sarif: Sarif, repoId: UUID) {
@@ -41,12 +54,15 @@ class ToolRunService(
val toolRun = run.asDbObject()
toolRun.tool = tool
toolRun.addRules(rules)
- toolRun.addToolResults(toolResults)
+ toolRun.addFindings(toolResults)
val result = toolRunRepository.save(toolRun)
toolRepository.save(tool)
repo.ifPresent {
+ if (!it.tools.contains(tool)) {
+ it.tools.add(tool)
+ }
it.addToolResult(result)
repositoryRepository.save(it)
}
@@ -54,9 +70,9 @@ class ToolRunService(
}
private fun createNewRulesAndSaveToTool(
- sarifRules: List<de.fraunhofer.iem.dataprovider.sarif.Rule>,
+ sarifRules: List<de.fraunhofer.iem.dataprovider.sarif.SarifRule>,
rules: MutableList<Rule>,
- tool: de.fraunhofer.iem.dataprovider.toolResult.Tool
+ tool: Tool
) {
sarifRules.forEach { sarifRule ->
if (rules.find { it.sarifRuleId == sarifRule.id } == null) {
@@ -67,7 +83,7 @@ class ToolRunService(
}
}
- private fun findOrCreateTool(tool: Tool): de.fraunhofer.iem.dataprovider.toolResult.Tool {
+ private fun findOrCreateTool(tool: SarifTool): Tool {
val fullName = tool.driver.fullName
val name = tool.driver.name
val version = tool.driver.version
@@ -83,4 +99,4 @@ class ToolRunService(
mutableListOf()
}
}
-}
\ No newline at end of file
+}
--
GitLab