updated caching for odc

introduced multiple tasks as groupTasks

src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/Sarif.kt

@@ -89,25 +89,25 @@ data class Tool(
 @Serializable
 data class Driver(
     @SerialName("downloadUri")
-    val downloadUri: String,
+    val downloadUri: String? = null,
     @SerialName("fullName")
-    val fullName: String,
+    val fullName: String? = null,
     @SerialName("guid")
-    val guid: String,
+    val guid: String? = null,
     @SerialName("informationUri")
-    val informationUri: String,
+    val informationUri: String? = null,
     @SerialName("language")
-    val language: String,
+    val language: String? = null,
     @SerialName("name")
-    val name: String,
+    val name: String? = null,
     @SerialName("organization")
-    val organization: String,
+    val organization: String? = null,
     @SerialName("rules")
     val rules: List<Rule>,
     @SerialName("semanticVersion")
-    val semanticVersion: String,
+    val semanticVersion: String? = null,
     @SerialName("version")
-    val version: String
+    val version: String? = null
 )
 
 @Serializable

src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt

@@ -22,7 +22,8 @@ fun de.fraunhofer.iem.dataprovider.sarif.Rule.asDbObject(): Rule {
 fun de.fraunhofer.iem.dataprovider.sarif.Tool.asDbObject(): Tool {
     val tool = Tool()
     tool.addRules(this.driver.rules.map { it.asDbObject() })
-    tool.name = this.driver.fullName
+    tool.fullName = this.driver.fullName
+    tool.name = this.driver.name
     tool.version = this.driver.version
     return tool
 }

src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt

@@ -4,10 +4,7 @@ import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
 import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
 import de.fraunhofer.iem.dataprovider.logger.getLogger
 import de.fraunhofer.iem.dataprovider.sarif.Sarif
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.CloneGitTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.DetektTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.GetGitlabProjectTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.GitRepository
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.*
 import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
 import jakarta.annotation.PreDestroy
 import kotlinx.coroutines.CoroutineScope
@@ -29,7 +26,10 @@ sealed class TaskDone : Event() {
 
 class GitCloneDone(override val taskId: UUID, val repoId: UUID, val outputDirectory: String) : TaskDone()
 class GetGitlabProjectDone(override val taskId: UUID, val repoId: UUID, val gitRepository: GitRepository) : TaskDone()
-class ProcessTaskDone(override val taskId: UUID, val message: String) : TaskDone()
+
+class SarifProcessDone(override val taskId: UUID, val repoId: UUID, val sarif: Sarif) : TaskDone()
+class SarifProcessGroupDone(override val taskId: UUID, val repoId: UUID, val groupId: UUID, val sarif: Sarif) :
+    TaskDone()
 
 class DetektDone(override val taskId: UUID, val sarif: Sarif) : TaskDone()
 
@@ -89,7 +89,7 @@ class TaskManager(
     private inner class MyActor {
 
         private val logger = getLogger(javaClass)
-
+        private val dependentEvents: MutableMap<UUID, MutableSet<UUID>> = mutableMapOf()
         suspend fun onReceive(event: Event) {
             logger.info("[${Thread.currentThread().name}] add task called in Task Manager $event")
 
@@ -118,19 +118,49 @@ class TaskManager(
                 }
 
                 is GitCloneDone -> {
+                    val taskIds = mutableSetOf<UUID>()
+                    val groupId = UUID.randomUUID()
                     // TODO: analysis results path should be unique or best not placed in the git folder
                     val analysisResultsPath = Paths.get(event.outputDirectory, "tool-results").toString()
-//                    worker.addTask(OdcTask(event.outputDirectory, analysisResultsPath, ::addEvent))
-                    worker.addTask(
-                        DetektTask(
-                            event.outputDirectory,
-                            analysisResultsPath,
-                            ::addEvent,
-                            event.repoId,
-                            repositoryRepository,
-                            toolRunRepository
-                        )
+
+                    val odcTask = OdcTask(
+                        event.outputDirectory,
+                        analysisResultsPath,
+                        ::addEvent,
+                        event.repoId,
+                        repositoryRepository,
+                        toolRunRepository, groupId
+                    )
+
+                    val detektTask = DetektTask(
+                        event.outputDirectory,
+                        analysisResultsPath,
+                        ::addEvent,
+                        event.repoId,
+                        repositoryRepository,
+                        toolRunRepository,
+                        groupID = groupId
                     )
+
+                    taskIds.add(odcTask.taskID)
+                    taskIds.add(detektTask.taskID)
+
+                    dependentEvents[groupId] = taskIds
+
+                    worker.addTask(odcTask)
+                    worker.addTask(detektTask)
+                }
+
+                is SarifProcessGroupDone -> {
+                    val eventGroup = dependentEvents[event.groupId]
+
+                    if (eventGroup != null) {
+                        eventGroup.remove(event.taskId)
+                        if (eventGroup.isEmpty()) {
+                            // TODO: Start metrics calculation
+                            worker.addTask(MetricsTask(event.repoId, repositoryRepository, ::addEvent))
+                        }
+                    }
                 }
 
 

src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt

@@ -2,8 +2,6 @@ package de.fraunhofer.iem.dataprovider.taskManager.tasks
 
 
 import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
-import de.fraunhofer.iem.dataprovider.sarif.Sarif
-import de.fraunhofer.iem.dataprovider.taskManager.DetektDone
 import de.fraunhofer.iem.dataprovider.taskManager.Event
 import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
 import org.springframework.core.io.ClassPathResource
@@ -14,19 +12,16 @@ import java.util.*
 
 
 class DetektTask(
-    projectPath: String, outputPath: String, override val responseChannel: suspend (task: Event) -> Unit,
+    projectPath: String, outputPath: String,
+    override val responseChannel: suspend (task: Event) -> Unit,
     override val repoId: UUID,
     override val repository: RepositoryRepository,
-    override val toolRunRepository: ToolRunRepository
+    override val toolRunRepository: ToolRunRepository,
+    override val groupID: UUID? = null
 ) : SarifTask() {
 
     private val resource: Resource = ClassPathResource("scripts/detekt.sh")
     override val flags: Array<String> = arrayOf(resource.file.absolutePath, projectPath, outputPath)
     override val execPath: String = "/bin/sh"
     override val resultPath: Path = Paths.get(outputPath, "detekt", "report.sarif")
-
-
-    override suspend fun sendResult(sarif: Sarif) {
-        responseChannel(DetektDone(this.taskID, sarif))
-    }
-}
\ No newline at end of file
+}

src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/MetricsTask.kt

+package de.fraunhofer.iem.dataprovider.taskManager.tasks
+
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
+import de.fraunhofer.iem.dataprovider.taskManager.Event
+import java.util.*
+
+class MetricsTask(
+    val repoId: UUID,
+    val repository: RepositoryRepository,
+    override val responseChannel: suspend (task: Event) -> Unit
+) : Task() {
+    override suspend fun execute() {
+//        val repo = repository.findById(repoId).get()
+//        repo.toolRuns.forEach {tr ->
+//
+//        }
+    }
+}
\ No newline at end of file

src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/OdcTask.kt

 package de.fraunhofer.iem.dataprovider.taskManager.tasks
 
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
 import de.fraunhofer.iem.dataprovider.taskManager.Event
-import de.fraunhofer.iem.dataprovider.taskManager.ProcessTaskDone
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
 import org.springframework.core.io.ClassPathResource
 import org.springframework.core.io.Resource
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.util.*
 
 // TODO: extend the existing odc setup to have an external db with cve lists.
 //  Else every docker run will load all indixes for a couple of minutes..
-class OdcTask(projectPath: String, outputPath: String, override val responseChannel: suspend (task: Event) -> Unit) :
-    ProcessTask() {
+class OdcTask(
+    projectPath: String, outputPath: String,
+    override val responseChannel: suspend (task: Event) -> Unit,
+    override val repoId: UUID,
+    override val repository: RepositoryRepository,
+    override val toolRunRepository: ToolRunRepository,
+    override val groupID: UUID? = null
+) : SarifTask() {
 
     private val resource: Resource = ClassPathResource("scripts/odc.sh")
     override val flags: Array<String> = arrayOf(resource.file.absolutePath, outputPath, projectPath)
-
     override val execPath: String = "/bin/sh"
-
-    override suspend fun handleProcessReturn(p: Process) {
-        logger.info(resource.toString())
-        val returnMessage = "Odc finished with exit code ${p.exitValue()}"
-        val output = String(p.inputStream.readAllBytes())
-        logger.info("Process output $output")
-        responseChannel(ProcessTaskDone(taskID, returnMessage))
-    }
+    override val resultPath: Path = Paths.get(outputPath, "odc", "dependency-check-report.sarif")
 }

src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt

@@ -4,11 +4,12 @@ import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
 import de.fraunhofer.iem.dataprovider.sarif.Sarif
 import de.fraunhofer.iem.dataprovider.sarif.asDbObject
 import de.fraunhofer.iem.dataprovider.sarif.getSarifFromFilePath
+import de.fraunhofer.iem.dataprovider.taskManager.SarifProcessDone
+import de.fraunhofer.iem.dataprovider.taskManager.SarifProcessGroupDone
 import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
-import org.springframework.transaction.annotation.Transactional
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -41,9 +42,10 @@ sealed class ProcessTask : Task() {
     }
 
     abstract suspend fun handleProcessReturn(p: Process)
+    abstract suspend fun sendResult(sarif: Sarif)
 
 }
-@Transactional
+
 sealed class SarifTask : ProcessTask() {
     protected abstract val repoId: UUID
     protected abstract val repository: RepositoryRepository
@@ -63,13 +65,20 @@ sealed class SarifTask : ProcessTask() {
         val results = toolRunRepository.saveAll(sarifDb)
 
         repo.ifPresent {
-//            Hibernate.initialize(it.toolRuns) // TODO: this fails because the session seems to be closed and add won't work because toolRuns are loaded lazily
             it.addToolResults(results)
             repository.save(it)
         }
         sendResult(sarifResult)
     }
 
-    abstract suspend fun sendResult(sarif: Sarif)
+    override suspend fun sendResult(sarif: Sarif) {
 
+        val event = if (groupID != null) {
+            SarifProcessGroupDone(this.taskID, repoId, groupID!!, sarif)
+        } else {
+            SarifProcessDone(this.taskID, repoId, sarif)
+        }
+
+        responseChannel(event)
+    }
 }

src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/Task.kt

@@ -11,6 +11,8 @@ interface ITask {
 
 sealed class Task : ITask {
     override val taskID: UUID = UUID.randomUUID()
+    protected open val groupID: UUID? = null
+
     protected val logger = getLogger(javaClass)
     protected abstract val responseChannel: suspend (task: Event) -> Unit
 }

src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt

@@ -13,7 +13,7 @@ enum class Level {
 }
 
 enum class Kind {
-    pass, open, informational, notApplicable
+    Pass, Open, Informational, NotApplicable
 }
 
 @Entity
@@ -59,6 +59,8 @@ class Tool {
     @Column(name = "id", nullable = false)
     var id: UUID? = null
 
+    @Column(name = "full_name", length = 500)
+    var fullName: String? = null
 
     @Column(name = "name", length = 500)
     var name: String? = null

src/main/resources/scripts/odc.sh

 #!/bin/sh
 
 DC_VERSION="latest"
-DC_DIRECTORY=${1}/OWASP-Dependency-Check
+DC_DIRECTORY=${1}/odc
 DC_PROJECT="dependency-check scan: ${2}"
-DATA_DIRECTORY="$DC_DIRECTORY/data"
-CACHE_DIRECTORY="$DC_DIRECTORY/data/cache"
+DATA_DIRECTORY="/tmp/opencode/data"
+CACHE_DIRECTORY="/tmp/opencode/data/cache"
 
 if [ ! -d "$DATA_DIRECTORY" ]; then
     echo "Initially creating persistent directory: $DATA_DIRECTORY"
@@ -21,11 +21,12 @@ fi
 docker run --rm \
     --volume "${2}":/src:z \
     --volume "$DATA_DIRECTORY":/usr/share/dependency-check/data:z \
-    --volume "$DC_DIRECTORY"/odc-reports:/report:z \
+    --volume "$DC_DIRECTORY":/odc:z \
     odc \
     --scan /src \
     --format "ALL" \
     --project "$DC_PROJECT" \
-    --out /report
+    --data /usr/share/dependency-check/data \
+    --out /odc
     # Use suppression like this: (where /src == $pwd)
     # --suppression "/src/security/dependency-check-suppression.xml"
\ No newline at end of file