From c0b5b8ad4222b65cc7dc5d46be14d3834b255d21 Mon Sep 17 00:00:00 2001
From: Jan-Niclas Struewer <j.n.struewer@gmail.com>
Date: Tue, 16 May 2023 09:06:45 +0200
Subject: [PATCH] added data persistence for sarif-based tools. Currently, not
working due to lazy loading issue with toolRuns in repository db object.
---
.../iem/dataprovider/gitlab/Repository.kt | 15 ++++++-
.../gitlab/RepositoryRepository.kt | 6 +++
.../dataprovider/taskManager/TaskManager.kt | 44 ++++++++++++++-----
.../taskManager/tasks/CloneGitTask.kt | 26 ++++++++---
.../taskManager/tasks/DetektTask.kt | 34 ++++++--------
.../taskManager/tasks/GitlabTask.kt | 16 +++++--
.../taskManager/tasks/ProcessTask.kt | 40 +++++++++++++++++
.../ToolResultsPersistenceService.kt | 17 -------
.../toolResult/ToolResultsRepository.kt | 4 --
.../iem/dataprovider/toolResult/ToolRun.kt | 16 ++++---
10 files changed, 147 insertions(+), 71 deletions(-)
delete mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsPersistenceService.kt
delete mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsRepository.kt
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
index 90e0d078..7fca2911 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/Repository.kt
@@ -14,6 +14,19 @@ class Repository {
var id: UUID? = null
@OrderBy("time_stamp DESC")
- @OneToMany(cascade = [CascadeType.ALL], mappedBy = "repository")
+ @OneToMany(mappedBy = "repository", cascade = [CascadeType.ALL])
var toolRuns: MutableList<ToolRun> = mutableListOf()
+
+ @Column(name = "name")
+ var name: String? = null
+
+ @Column(name = "repo_id")
+ var repoId: Long? = null
+
+ @Enumerated
+ @Column(name = "platform")
+ var platform: Platform? = null
+
+ @Column(name = "url")
+ var url: String? = null
}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt
index 3e39deed..23fa8211 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/RepositoryRepository.kt
@@ -4,4 +4,10 @@ import org.springframework.data.jpa.repository.JpaRepository
import java.util.*
interface RepositoryRepository : JpaRepository<Repository, UUID> {
+
+
+ fun existsByRepoIdAndPlatform(repoId: Long, platform: Platform): Boolean
+
+
+ fun findByRepoIdAndPlatform(repoId: Long, platform: Platform): Repository?
}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
index e09f769a..c6488980 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
@@ -1,10 +1,14 @@
package de.fraunhofer.iem.dataprovider.taskManager
import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.sarif.Sarif
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.*
-import de.fraunhofer.iem.dataprovider.toolResult.ToolResultsPersistenceService
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.CloneGitTask
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.DetektTask
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.GetGitlabProjectTask
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.GitRepository
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import jakarta.annotation.PreDestroy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -23,8 +27,8 @@ sealed class TaskDone : Event() {
abstract val taskId: UUID
}
-class GitCloneDone(override val taskId: UUID, val outputDirectory: String) : TaskDone()
-class GetGitlabProjectDone(override val taskId: UUID, val gitProject: GitProject) : TaskDone()
+class GitCloneDone(override val taskId: UUID, val repoId: UUID, val outputDirectory: String) : TaskDone()
+class GetGitlabProjectDone(override val taskId: UUID, val repoId: UUID, val gitRepository: GitRepository) : TaskDone()
class ProcessTaskDone(override val taskId: UUID, val message: String) : TaskDone()
class DetektDone(override val taskId: UUID, val sarif: Sarif) : TaskDone()
@@ -37,7 +41,8 @@ class DetektDone(override val taskId: UUID, val sarif: Sarif) : TaskDone()
@Component
class TaskManager(
private val config: Config,
- private val toolResultsPersistenceService: ToolResultsPersistenceService
+ private val repositoryRepository: RepositoryRepository,
+ private val toolRunRepository: ToolRunRepository
) {
// The used default dispatcher is ok for CPU-bound workloads. However,
@@ -90,13 +95,22 @@ class TaskManager(
when (event) {
is RepoChangedEvent -> {
- ioWorker.addTask(GetGitlabProjectTask(event.repoId, event.gitConfiguration, ::addEvent))
+ ioWorker.addTask(
+ GetGitlabProjectTask(
+ event.repoId,
+ event.gitConfiguration,
+ ::addEvent,
+ repositoryRepository
+ )
+ )
}
is GetGitlabProjectDone -> {
+
ioWorker.addTask(
CloneGitTask(
- event.gitProject,
+ event.gitRepository,
+ event.repoId,
::addEvent,
config.gitProjectPath
)
@@ -106,13 +120,19 @@ class TaskManager(
is GitCloneDone -> {
// TODO: analysis results path should be unique or best not placed in the git folder
val analysisResultsPath = Paths.get(event.outputDirectory, "tool-results").toString()
- worker.addTask(OdcTask(event.outputDirectory, analysisResultsPath, ::addEvent))
- worker.addTask(DetektTask(event.outputDirectory, analysisResultsPath, ::addEvent))
+// worker.addTask(OdcTask(event.outputDirectory, analysisResultsPath, ::addEvent))
+ worker.addTask(
+ DetektTask(
+ event.outputDirectory,
+ analysisResultsPath,
+ ::addEvent,
+ event.repoId,
+ repositoryRepository,
+ toolRunRepository
+ )
+ )
}
- is DetektDone -> {
- toolResultsPersistenceService.save(event.sarif)
- }
else -> {
logger.info("Received event without special handling associated $event")
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/CloneGitTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/CloneGitTask.kt
index 3cd1753c..187f4aeb 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/CloneGitTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/CloneGitTask.kt
@@ -1,33 +1,45 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
+import de.fraunhofer.iem.dataprovider.gitlab.Platform
+import de.fraunhofer.iem.dataprovider.gitlab.Repository
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.taskManager.GitCloneDone
import org.eclipse.jgit.api.Git
import java.nio.file.Paths
+import java.util.*
-data class GitProject(val name: String, val uri: String)
+data class GitRepository(val name: String, val uri: String, val id: Long, val platform: Platform)
+
+fun GitRepository.toDbObject(): Repository {
+ val repo = Repository()
+ repo.name = this.name
+ repo.repoId = this.id
+ repo.platform = this.platform
+ return repo
+}
class CloneGitTask(
- private val gitProject: GitProject,
+ private val gitRepository: GitRepository,
+ private val repoDbId: UUID,
override val responseChannel: suspend (task: Event) -> Unit,
private val outputPath: String,
) : Task() {
override suspend fun execute() {
- val outputDirName = "${gitProject.name}-${taskID}"
+ val outputDirName = "${gitRepository.name}-${taskID}"
val outputDirectory = Paths.get(outputPath, outputDirName)
- logger.info("Cloning ${gitProject.name} into $outputDirectory")
+ logger.info("Cloning ${gitRepository.name} into $outputDirectory")
val git: Git = Git.cloneRepository()
- .setURI(gitProject.uri)
+ .setURI(gitRepository.uri)
.setDirectory(outputDirectory.toFile())
.call()
git.close()
- responseChannel(GitCloneDone(taskID, outputDirectory.toString()))
+ responseChannel(GitCloneDone(taskID, repoDbId, outputDirectory.toString()))
- logger.info("Finished cloning ${gitProject.name}")
+ logger.info("Finished cloning ${gitRepository.name}")
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
index 1d464e8d..18d46f47 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
@@ -1,38 +1,32 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
-import de.fraunhofer.iem.dataprovider.sarif.getSarifFromFilePath
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
+import de.fraunhofer.iem.dataprovider.sarif.Sarif
import de.fraunhofer.iem.dataprovider.taskManager.DetektDone
import de.fraunhofer.iem.dataprovider.taskManager.Event
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.Resource
+import java.nio.file.Path
import java.nio.file.Paths
+import java.util.*
-class DetektTask(projectPath: String, outputPath: String, override val responseChannel: suspend (task: Event) -> Unit) :
- ProcessTask() {
+class DetektTask(
+ projectPath: String, outputPath: String, override val responseChannel: suspend (task: Event) -> Unit,
+ override val repoId: UUID,
+ override val repository: RepositoryRepository,
+ override val toolRunRepository: ToolRunRepository
+) : SarifTask() {
private val resource: Resource = ClassPathResource("scripts/detekt.sh")
override val flags: Array<String> = arrayOf(resource.file.absolutePath, projectPath, outputPath)
-
override val execPath: String = "/bin/sh"
+ override val resultPath: Path = Paths.get(outputPath, "detekt", "report.sarif")
- private val resultPath = Paths.get(outputPath, "detekt", "report.sarif")
- override suspend fun handleProcessReturn(p: Process) {
- logger.info(resource.toString())
-
- val returnMessage = "Odc finished with exit code ${p.exitValue()}"
-// val output = String(p.inputStream.readAllBytes())
-// logger.info("Process output $output")
-
- val sarifResult = getSarifFromFilePath(resultPath)
-
- sarifResult.runs.forEach {
- it.results.forEach {
- logger.info("run in sarif result message: ${it.message}")
- }
- }
- responseChannel(DetektDone(this.taskID, sarifResult))
+ override suspend fun sendResult(sarif: Sarif) {
+ responseChannel(DetektDone(this.taskID, sarif))
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt
index 7ac7a188..3a465cbe 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GitlabTask.kt
@@ -1,6 +1,8 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
+import de.fraunhofer.iem.dataprovider.gitlab.Platform
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.taskManager.GetGitlabProjectDone
import org.gitlab4j.api.GitLabApi
@@ -8,7 +10,8 @@ import org.gitlab4j.api.GitLabApi
class GetGitlabProjectTask(
private val repoId: Long, private val gitlabConfiguration: GitConfiguration,
- override val responseChannel: suspend (task: Event) -> Unit
+ override val responseChannel: suspend (task: Event) -> Unit,
+ private val repository: RepositoryRepository
) : Task() {
@@ -20,10 +23,17 @@ class GetGitlabProjectTask(
val project = gitlabApi.projectApi.getProject(repoId)
logger.info(project.toString())
val projectUri = project.httpUrlToRepo
- val gitProject = GitProject(project.path, projectUri)
- responseChannel(GetGitlabProjectDone(taskID, gitProject))
+ val gitRepository = GitRepository(project.path, projectUri, repoId, Platform.OPEN_CODE)
+
+ var repo = repository.findByRepoIdAndPlatform(gitRepository.id, gitRepository.platform)
+ if (repo == null) {
+ logger.info("Repository ${gitRepository.id} ${gitRepository.platform} doesn't exist in database.")
+ repo = repository.save(gitRepository.toDbObject())
+ }
logger.info("Retrieved project ${project.path} and url $projectUri")
+
+ responseChannel(GetGitlabProjectDone(taskID, repo.id!!, gitRepository))
}
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
index 46c6a45a..b27c4c58 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
@@ -1,10 +1,19 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
+import de.fraunhofer.iem.dataprovider.sarif.Sarif
+import de.fraunhofer.iem.dataprovider.sarif.asDbObject
+import de.fraunhofer.iem.dataprovider.sarif.getSarifFromFilePath
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
+import org.hibernate.Hibernate
+import org.springframework.transaction.annotation.Transactional
import java.nio.file.Files
+import java.nio.file.Path
import java.nio.file.Paths
+import java.util.*
sealed class ProcessTask : Task() {
protected abstract val flags: Array<String>
@@ -33,4 +42,35 @@ sealed class ProcessTask : Task() {
}
abstract suspend fun handleProcessReturn(p: Process)
+
+}
+
+sealed class SarifTask : ProcessTask() {
+ protected abstract val repoId: UUID
+ protected abstract val repository: RepositoryRepository
+ protected abstract val toolRunRepository: ToolRunRepository
+ protected abstract val resultPath: Path
+
+ @Transactional
+ override suspend fun handleProcessReturn(p: Process) {
+
+ // TODO: check process exit codes. Current problem, if an analysis
+ // has findings it might return an exit code != 0 even tho the process finished correctly
+
+ logger.info("Handle Process return in $javaClass")
+ val sarifResult = getSarifFromFilePath(resultPath)
+ val repo = repository.findById(repoId)
+
+ val results = toolRunRepository.saveAll(sarifResult.asDbObject())
+
+ repo.ifPresent {
+ Hibernate.initialize(it.toolRuns) // TODO: this fails because the session seems to be closed and add won't work because toolRuns are loaded lazily
+ it.toolRuns.addAll(results)
+ repository.save(it)
+ }
+ sendResult(sarifResult)
+ }
+
+ abstract suspend fun sendResult(sarif: Sarif)
+
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsPersistenceService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsPersistenceService.kt
deleted file mode 100644
index c828d707..00000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsPersistenceService.kt
+++ /dev/null
@@ -1,17 +0,0 @@
-package de.fraunhofer.iem.dataprovider.toolResult
-
-import de.fraunhofer.iem.dataprovider.logger.getLogger
-import de.fraunhofer.iem.dataprovider.sarif.Sarif
-import org.springframework.stereotype.Service
-
-@Service
-class ToolResultsPersistenceService(val toolRunRepository: ToolRunRepository) {
-
- private val logger = getLogger(javaClass)
- suspend fun save(sarif: Sarif) {
- logger.info("Save sarif request received.")
-
- toolRunRepository.save(ToolRun())
- }
-
-}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsRepository.kt
deleted file mode 100644
index e878abc9..00000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolResultsRepository.kt
+++ /dev/null
@@ -1,4 +0,0 @@
-package de.fraunhofer.iem.dataprovider.toolResult
-
-//interface ToolResultsRepository : CoroutineCrudRepository<ToolResultsDb, String> {
-//}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
index d8ad7dbe..1c39b55d 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
@@ -20,10 +20,10 @@ class ToolRun {
@Column(name = "id", nullable = false)
var id: UUID? = null
- @OneToMany(mappedBy = "toolRun", orphanRemoval = true)
+ @OneToMany(mappedBy = "toolRun", orphanRemoval = true, cascade = [CascadeType.ALL])
var toolResults: MutableSet<ToolResult> = mutableSetOf()
- @OneToOne(orphanRemoval = true)
+ @OneToOne(orphanRemoval = true, cascade = [CascadeType.ALL])
@JoinColumn(name = "tool_id")
var tool: Tool? = null
@@ -44,10 +44,10 @@ class Tool {
@Column(name = "id", nullable = false)
var id: UUID? = null
- @OneToMany(mappedBy = "tool", orphanRemoval = true)
+ @OneToMany(mappedBy = "tool", orphanRemoval = true, cascade = [CascadeType.ALL])
var rules: MutableSet<Rule> = mutableSetOf()
- @Column(name = "name")
+ @Column(name = "name", length = 500)
var name: String? = null
@Column(name = "version")
@@ -74,7 +74,7 @@ class ToolResult {
@JoinColumn(name = "tool_run_id")
var toolRun: ToolRun? = null
- @OneToMany(mappedBy = "toolResult", orphanRemoval = true)
+ @OneToMany(mappedBy = "toolResult", cascade = [CascadeType.ALL], orphanRemoval = true)
var resultLocations: MutableSet<ResultLocation> = mutableSetOf()
@Enumerated
@@ -85,6 +85,7 @@ class ToolResult {
@Column(name = "kind")
var kind: Kind? = null
+ @Lob
@Column(name = "message")
var message: String? = null
@@ -115,6 +116,7 @@ class ResultLocation {
@JoinColumn(name = "rule_id")
var rule: Rule? = null
+ @Lob
@Column(name = "artifact_location")
var artifactLocation: String? = null
@@ -157,14 +159,14 @@ class Rule {
@Column(name = "rule_id")
var ruleId: String? = null
-
+ @Lob
@Column(name = "short_description")
var shortDescription: String? = null
@ElementCollection
@CollectionTable(name = "rule_messages", joinColumns = [JoinColumn(name = "owner_id")])
- @Column(name = "message")
+ @Column(name = "message", length = 1000)
var messages: MutableList<String> = mutableListOf()
override fun equals(other: Any?): Boolean {
--
GitLab