diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/Sarif.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/Sarif.kt
index dfc4a8a7706884ee98ff4b411fd3dcf1709aca0e..7768af1419bef7d5f1dd0cb430652a3df76e5096 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/Sarif.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/Sarif.kt
@@ -89,25 +89,25 @@ data class Tool(
@Serializable
data class Driver(
@SerialName("downloadUri")
- val downloadUri: String,
+ val downloadUri: String? = null,
@SerialName("fullName")
- val fullName: String,
+ val fullName: String? = null,
@SerialName("guid")
- val guid: String,
+ val guid: String? = null,
@SerialName("informationUri")
- val informationUri: String,
+ val informationUri: String? = null,
@SerialName("language")
- val language: String,
+ val language: String? = null,
@SerialName("name")
- val name: String,
+ val name: String? = null,
@SerialName("organization")
- val organization: String,
+ val organization: String? = null,
@SerialName("rules")
val rules: List<Rule>,
@SerialName("semanticVersion")
- val semanticVersion: String,
+ val semanticVersion: String? = null,
@SerialName("version")
- val version: String
+ val version: String? = null
)
@Serializable
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
index bf90606284b159bcb5d01d5cb6cdec6868a3b90a..b535c2e3c286bd0bf89226200130319ac14cc9d8 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/sarif/SarifExtensions.kt
@@ -22,7 +22,8 @@ fun de.fraunhofer.iem.dataprovider.sarif.Rule.asDbObject(): Rule {
fun de.fraunhofer.iem.dataprovider.sarif.Tool.asDbObject(): Tool {
val tool = Tool()
tool.addRules(this.driver.rules.map { it.asDbObject() })
- tool.name = this.driver.fullName
+ tool.fullName = this.driver.fullName
+ tool.name = this.driver.name
tool.version = this.driver.version
return tool
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
index c6488980bf5f51bf85325a6779cdebbe2c598b30..d2fda8fab65e3ee16c162e38024284fe3da534fd 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
@@ -4,10 +4,7 @@ import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.sarif.Sarif
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.CloneGitTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.DetektTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.GetGitlabProjectTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.GitRepository
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.*
import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import jakarta.annotation.PreDestroy
import kotlinx.coroutines.CoroutineScope
@@ -29,7 +26,10 @@ sealed class TaskDone : Event() {
class GitCloneDone(override val taskId: UUID, val repoId: UUID, val outputDirectory: String) : TaskDone()
class GetGitlabProjectDone(override val taskId: UUID, val repoId: UUID, val gitRepository: GitRepository) : TaskDone()
-class ProcessTaskDone(override val taskId: UUID, val message: String) : TaskDone()
+
+class SarifProcessDone(override val taskId: UUID, val repoId: UUID, val sarif: Sarif) : TaskDone()
+class SarifProcessGroupDone(override val taskId: UUID, val repoId: UUID, val groupId: UUID, val sarif: Sarif) :
+ TaskDone()
class DetektDone(override val taskId: UUID, val sarif: Sarif) : TaskDone()
@@ -89,7 +89,7 @@ class TaskManager(
private inner class MyActor {
private val logger = getLogger(javaClass)
-
+ private val dependentEvents: MutableMap<UUID, MutableSet<UUID>> = mutableMapOf()
suspend fun onReceive(event: Event) {
logger.info("[${Thread.currentThread().name}] add task called in Task Manager $event")
@@ -118,19 +118,49 @@ class TaskManager(
}
is GitCloneDone -> {
+ val taskIds = mutableSetOf<UUID>()
+ val groupId = UUID.randomUUID()
// TODO: analysis results path should be unique or best not placed in the git folder
val analysisResultsPath = Paths.get(event.outputDirectory, "tool-results").toString()
-// worker.addTask(OdcTask(event.outputDirectory, analysisResultsPath, ::addEvent))
- worker.addTask(
- DetektTask(
- event.outputDirectory,
- analysisResultsPath,
- ::addEvent,
- event.repoId,
- repositoryRepository,
- toolRunRepository
- )
+
+ val odcTask = OdcTask(
+ event.outputDirectory,
+ analysisResultsPath,
+ ::addEvent,
+ event.repoId,
+ repositoryRepository,
+ toolRunRepository, groupId
+ )
+
+ val detektTask = DetektTask(
+ event.outputDirectory,
+ analysisResultsPath,
+ ::addEvent,
+ event.repoId,
+ repositoryRepository,
+ toolRunRepository,
+ groupID = groupId
)
+
+ taskIds.add(odcTask.taskID)
+ taskIds.add(detektTask.taskID)
+
+ dependentEvents[groupId] = taskIds
+
+ worker.addTask(odcTask)
+ worker.addTask(detektTask)
+ }
+
+ is SarifProcessGroupDone -> {
+ val eventGroup = dependentEvents[event.groupId]
+
+ if (eventGroup != null) {
+ eventGroup.remove(event.taskId)
+ if (eventGroup.isEmpty()) {
+ // TODO: Start metrics calculation
+ worker.addTask(MetricsTask(event.repoId, repositoryRepository, ::addEvent))
+ }
+ }
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
index 18d46f475692b94bf787c7a36b30bafa89c504ea..af6ec9ae60b0d581b0d0c09cfe4b2cd7953cb70d 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/DetektTask.kt
@@ -2,8 +2,6 @@ package de.fraunhofer.iem.dataprovider.taskManager.tasks
import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
-import de.fraunhofer.iem.dataprovider.sarif.Sarif
-import de.fraunhofer.iem.dataprovider.taskManager.DetektDone
import de.fraunhofer.iem.dataprovider.taskManager.Event
import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import org.springframework.core.io.ClassPathResource
@@ -14,19 +12,16 @@ import java.util.*
class DetektTask(
- projectPath: String, outputPath: String, override val responseChannel: suspend (task: Event) -> Unit,
+ projectPath: String, outputPath: String,
+ override val responseChannel: suspend (task: Event) -> Unit,
override val repoId: UUID,
override val repository: RepositoryRepository,
- override val toolRunRepository: ToolRunRepository
+ override val toolRunRepository: ToolRunRepository,
+ override val groupID: UUID? = null
) : SarifTask() {
private val resource: Resource = ClassPathResource("scripts/detekt.sh")
override val flags: Array<String> = arrayOf(resource.file.absolutePath, projectPath, outputPath)
override val execPath: String = "/bin/sh"
override val resultPath: Path = Paths.get(outputPath, "detekt", "report.sarif")
-
-
- override suspend fun sendResult(sarif: Sarif) {
- responseChannel(DetektDone(this.taskID, sarif))
- }
-}
\ No newline at end of file
+}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/MetricsTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/MetricsTask.kt
new file mode 100644
index 0000000000000000000000000000000000000000..1d07ca14618185d9f56eaf110595c9827572f0da
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/MetricsTask.kt
@@ -0,0 +1,18 @@
+package de.fraunhofer.iem.dataprovider.taskManager.tasks
+
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
+import de.fraunhofer.iem.dataprovider.taskManager.Event
+import java.util.*
+
+class MetricsTask(
+ val repoId: UUID,
+ val repository: RepositoryRepository,
+ override val responseChannel: suspend (task: Event) -> Unit
+) : Task() {
+ override suspend fun execute() {
+// val repo = repository.findById(repoId).get()
+// repo.toolRuns.forEach {tr ->
+//
+// }
+ }
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/OdcTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/OdcTask.kt
index 36e1994ffbb9c35238ccc418ffaae18101aba7b1..12c653c5607210680322c0eba7f1cc6c0792e22d 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/OdcTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/OdcTask.kt
@@ -1,25 +1,27 @@
package de.fraunhofer.iem.dataprovider.taskManager.tasks
+import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.taskManager.Event
-import de.fraunhofer.iem.dataprovider.taskManager.ProcessTaskDone
+import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import org.springframework.core.io.ClassPathResource
import org.springframework.core.io.Resource
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.util.*
// TODO: extend the existing odc setup to have an external db with cve lists.
// Else every docker run will load all indixes for a couple of minutes..
-class OdcTask(projectPath: String, outputPath: String, override val responseChannel: suspend (task: Event) -> Unit) :
- ProcessTask() {
+class OdcTask(
+ projectPath: String, outputPath: String,
+ override val responseChannel: suspend (task: Event) -> Unit,
+ override val repoId: UUID,
+ override val repository: RepositoryRepository,
+ override val toolRunRepository: ToolRunRepository,
+ override val groupID: UUID? = null
+) : SarifTask() {
private val resource: Resource = ClassPathResource("scripts/odc.sh")
override val flags: Array<String> = arrayOf(resource.file.absolutePath, outputPath, projectPath)
-
override val execPath: String = "/bin/sh"
-
- override suspend fun handleProcessReturn(p: Process) {
- logger.info(resource.toString())
- val returnMessage = "Odc finished with exit code ${p.exitValue()}"
- val output = String(p.inputStream.readAllBytes())
- logger.info("Process output $output")
- responseChannel(ProcessTaskDone(taskID, returnMessage))
- }
+ override val resultPath: Path = Paths.get(outputPath, "odc", "dependency-check-report.sarif")
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
index b9fe3691b0d954200e131faa1e01cdea0969c338..f1793de1f065d90848bb39b5362d9a139fa34b75 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/ProcessTask.kt
@@ -4,11 +4,12 @@ import de.fraunhofer.iem.dataprovider.gitlab.RepositoryRepository
import de.fraunhofer.iem.dataprovider.sarif.Sarif
import de.fraunhofer.iem.dataprovider.sarif.asDbObject
import de.fraunhofer.iem.dataprovider.sarif.getSarifFromFilePath
+import de.fraunhofer.iem.dataprovider.taskManager.SarifProcessDone
+import de.fraunhofer.iem.dataprovider.taskManager.SarifProcessGroupDone
import de.fraunhofer.iem.dataprovider.toolResult.ToolRunRepository
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
-import org.springframework.transaction.annotation.Transactional
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
@@ -41,9 +42,10 @@ sealed class ProcessTask : Task() {
}
abstract suspend fun handleProcessReturn(p: Process)
+ abstract suspend fun sendResult(sarif: Sarif)
}
-@Transactional
+
sealed class SarifTask : ProcessTask() {
protected abstract val repoId: UUID
protected abstract val repository: RepositoryRepository
@@ -63,13 +65,20 @@ sealed class SarifTask : ProcessTask() {
val results = toolRunRepository.saveAll(sarifDb)
repo.ifPresent {
-// Hibernate.initialize(it.toolRuns) // TODO: this fails because the session seems to be closed and add won't work because toolRuns are loaded lazily
it.addToolResults(results)
repository.save(it)
}
sendResult(sarifResult)
}
- abstract suspend fun sendResult(sarif: Sarif)
+ override suspend fun sendResult(sarif: Sarif) {
+ val event = if (groupID != null) {
+ SarifProcessGroupDone(this.taskID, repoId, groupID!!, sarif)
+ } else {
+ SarifProcessDone(this.taskID, repoId, sarif)
+ }
+
+ responseChannel(event)
+ }
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/Task.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/Task.kt
index 04bf166262c17622be570881010a6f899a2efc10..c029c4961bb9d885f976ffc1738bfe1a437c3209 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/Task.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/Task.kt
@@ -11,6 +11,8 @@ interface ITask {
sealed class Task : ITask {
override val taskID: UUID = UUID.randomUUID()
+ protected open val groupID: UUID? = null
+
protected val logger = getLogger(javaClass)
protected abstract val responseChannel: suspend (task: Event) -> Unit
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
index ed71bb931b55d70852c927aecc7ea62325be1338..7b947239769dc9c0ea0d9f26d514828c15322926 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolResult/ToolRun.kt
@@ -13,7 +13,7 @@ enum class Level {
}
enum class Kind {
- pass, open, informational, notApplicable
+ Pass, Open, Informational, NotApplicable
}
@Entity
@@ -59,6 +59,8 @@ class Tool {
@Column(name = "id", nullable = false)
var id: UUID? = null
+ @Column(name = "full_name", length = 500)
+ var fullName: String? = null
@Column(name = "name", length = 500)
var name: String? = null
diff --git a/src/main/resources/scripts/odc.sh b/src/main/resources/scripts/odc.sh
index 9c41229169bd4c8beaae262273e02c0ee6f6e318..46c5c5d54ed286dab4b3a199e29b20cf38fc4109 100755
--- a/src/main/resources/scripts/odc.sh
+++ b/src/main/resources/scripts/odc.sh
@@ -1,10 +1,10 @@
#!/bin/sh
DC_VERSION="latest"
-DC_DIRECTORY=${1}/OWASP-Dependency-Check
+DC_DIRECTORY=${1}/odc
DC_PROJECT="dependency-check scan: ${2}"
-DATA_DIRECTORY="$DC_DIRECTORY/data"
-CACHE_DIRECTORY="$DC_DIRECTORY/data/cache"
+DATA_DIRECTORY="/tmp/opencode/data"
+CACHE_DIRECTORY="/tmp/opencode/data/cache"
if [ ! -d "$DATA_DIRECTORY" ]; then
echo "Initially creating persistent directory: $DATA_DIRECTORY"
@@ -21,11 +21,12 @@ fi
docker run --rm \
--volume "${2}":/src:z \
--volume "$DATA_DIRECTORY":/usr/share/dependency-check/data:z \
- --volume "$DC_DIRECTORY"/odc-reports:/report:z \
+ --volume "$DC_DIRECTORY":/odc:z \
odc \
--scan /src \
--format "ALL" \
--project "$DC_PROJECT" \
- --out /report
+ --data /usr/share/dependency-check/data \
+ --out /odc
# Use suppression like this: (where /src == $pwd)
# --suppression "/src/security/dependency-check-suppression.xml"
\ No newline at end of file