From 531963936d4a6a6f56748a2309ed1313889bcb55 Mon Sep 17 00:00:00 2001
From: Jan-Niclas Struewer <j.n.struewer@gmail.com>
Date: Tue, 26 Sep 2023 16:56:48 +0200
Subject: [PATCH] implemented first integration of occmd tool
---
.../configuration/DirectoryPathsProperties.kt | 6 +-
.../kpi/metrics/MetricsService.kt | 15 +-
.../dataprovider/kpi/service/KPIService.kt | 80 +++--------
.../tasks/tools/occmd/OccmdTask.kt | 4 +-
.../tools/occmd/json/sarif/OccmdSarif.kt | 46 ------
.../tools/occmd/dto/OccmdKpiDto.kt | 9 ++
.../tools/occmd/entity/OccmdSecretEntity.kt | 33 -----
.../tools/occmd/enumeration/Checks.kt | 2 +-
.../raw => tools/occmd/json}/RawResultJson.kt | 2 +-
.../occmd/json}/SecretResultsJson.kt | 2 +-
.../occmd/json}/ToolSecretJson.kt | 2 +-
.../occmd/json}/ToolSecretResultJson.kt | 2 +-
.../tools/occmd/service/OccmdService.kt | 131 ++++++++++++++++++
src/main/resources/application.properties | 10 +-
src/main/resources/scripts/occmd.sh | 6 +-
src/main/resources/scripts/ort/config.yml | 5 -
src/main/resources/scripts/ort/ort_advisor.sh | 16 ---
.../resources/scripts/ort/ort_analyzer.sh | 16 ---
18 files changed, 194 insertions(+), 193 deletions(-)
delete mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/sarif/OccmdSarif.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/dto/OccmdKpiDto.kt
delete mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/entity/OccmdSecretEntity.kt
rename src/main/kotlin/de/fraunhofer/iem/dataprovider/{taskManager/tasks => }/tools/occmd/enumeration/Checks.kt (79%)
rename src/main/kotlin/de/fraunhofer/iem/dataprovider/{taskManager/tasks/tools/occmd/json/raw => tools/occmd/json}/RawResultJson.kt (84%)
rename src/main/kotlin/de/fraunhofer/iem/dataprovider/{taskManager/tasks/tools/occmd/json/raw => tools/occmd/json}/SecretResultsJson.kt (73%)
rename src/main/kotlin/de/fraunhofer/iem/dataprovider/{taskManager/tasks/tools/occmd/json/raw => tools/occmd/json}/ToolSecretJson.kt (76%)
rename src/main/kotlin/de/fraunhofer/iem/dataprovider/{taskManager/tasks/tools/occmd/json/raw => tools/occmd/json}/ToolSecretResultJson.kt (76%)
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt
delete mode 100644 src/main/resources/scripts/ort/config.yml
delete mode 100755 src/main/resources/scripts/ort/ort_advisor.sh
delete mode 100755 src/main/resources/scripts/ort/ort_analyzer.sh
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/configuration/DirectoryPathsProperties.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/configuration/DirectoryPathsProperties.kt
index 802f0f61..951661e2 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/configuration/DirectoryPathsProperties.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/configuration/DirectoryPathsProperties.kt
@@ -9,9 +9,13 @@ import java.nio.file.Path
import java.nio.file.Paths
import kotlin.io.path.createDirectories
-@ConfigurationProperties(prefix = "directories")
+@ConfigurationProperties(prefix = "occmd")
@Validated
data class DirectoryPathsProperties(
+ @NotBlank
+ @Length(min = 1)
+ val occmdPath: String,
+
@NotBlank
@Length(min = 1)
val gitCloneTargetDirectory: String,
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt
index 718e9071..a27d0af6 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt
@@ -4,6 +4,7 @@ import de.fraunhofer.iem.dataprovider.kpi.metrics.repositoryDetails.service.Repo
import de.fraunhofer.iem.dataprovider.kpi.service.KPIService
import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService
import de.fraunhofer.iem.dataprovider.toolRun.service.ToolRunService
+import de.fraunhofer.iem.dataprovider.tools.occmd.service.OccmdService
import de.fraunhofer.iem.dataprovider.tools.ort.service.OrtService
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -17,7 +18,8 @@ class MetricsService(
private val toolRunService: ToolRunService,
private val repositoryDetailsService: RepositoryDetailsService,
private val kpiService: KPIService,
- private val ortService: OrtService
+ private val ortService: OrtService,
+ private val occmdService: OccmdService
) {
private val defaultScope = CoroutineScope(Dispatchers.Default)
@@ -63,16 +65,20 @@ class MetricsService(
*/
val vulnerabilityKpis = async {
- val vulnerabilityDtos = ortService.getOrtResults(repoId)
+ val vulnerabilityDtos = ortService.getOrtResults(106) // in the dev setup we get results for repo id 106
ortService.calculateVulnerabilityKpis(vulnerabilityDtos)
}
-
val repositoryDetailsKpi = async {
val repoDetailsDto = repositoryDetailsService.getRepositoryDetails(repoId)
repositoryDetailsService.calculateRepositoryDetailsKpis(repoDetailsDto)
}
+ val occmdKpis = async {
+ val rawOccmdResults = occmdService.runOccmd(repoId, toolRun.repository?.url!!)
+ occmdService.calculateOccmdKpis(rawOccmdResults)
+ }
+
/**
* Calculate KPI tree by awaiting all tool KPIs
* joinAll(toolKpi)
@@ -81,7 +87,8 @@ class MetricsService(
kpiService.calculateAndSaveKpiTree(
repository = toolRun.repository!!,
repositoryDetailsKpi = repositoryDetailsKpi.await(),
- vulnerabilityKpis = vulnerabilityKpis.await()
+ vulnerabilityKpis = vulnerabilityKpis.await(),
+ occmdKpis = occmdKpis.await()
)
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt
index 63a5ee35..70e5f98c 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt
@@ -11,6 +11,7 @@ import de.fraunhofer.iem.dataprovider.kpi.strategy.MaximumKPICalculationStrategy
import de.fraunhofer.iem.dataprovider.kpi.strategy.RatioKPICalculationStrategy
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.repository.entity.RepositoryEntity
+import de.fraunhofer.iem.dataprovider.tools.occmd.dto.OccmdKpisDto
import org.springframework.data.repository.findByIdOrNull
import org.springframework.stereotype.Service
import java.util.*
@@ -26,10 +27,14 @@ class KPIService(
fun calculateAndSaveKpiTree(
repository: RepositoryEntity,
repositoryDetailsKpi: RepositoryDetailsKpisDto,
- vulnerabilityKpis: List<KPICreateDto>
+ vulnerabilityKpis: List<KPICreateDto>,
+ occmdKpis: OccmdKpisDto
) {
- val rootKPI =
- generateKPITree(repositoryDetailsKpi = repositoryDetailsKpi, vulnerabilityKpis = vulnerabilityKpis)
+ val rootKPI = generateKPITree(
+ repositoryDetailsKpi = repositoryDetailsKpi,
+ vulnerabilityKpis = vulnerabilityKpis,
+ occmdKpis = occmdKpis
+ )
calculateKPIs(rootKPI)
storeAndPurgeOld(repository, rootKPI)
}
@@ -37,7 +42,8 @@ class KPIService(
private fun generateKPITree(
repositoryDetailsKpi: RepositoryDetailsKpisDto,
- vulnerabilityKpis: List<KPICreateDto>
+ vulnerabilityKpis: List<KPICreateDto>,
+ occmdKpis: OccmdKpisDto
): KPICreateDto {
@@ -86,27 +92,19 @@ class KPIService(
)
vulnerabilityKpis.forEach { maximalDependencyVulnerabilityKPI.addChildKPI(it, 0.0) }
- securityKPI.addChildKPI(maximalDependencyVulnerabilityKPI, 0.5)
- //TODO: this code will be called from the dependency api request task !
-// val dependencyDtos = dependencyService.getDependenciesForRepository(repoId)
-// // We change the structure to associate vulnerabilities with dependencies instead of dependencies with vulnerabilities
-// val vulnerabilitiesWithDependencies = restructureDependenciesAndVulnerabilities(dependencyDtos)
-// // For each vulnerability, we create a sub raw data kpi
-// addMaximalDependencyVulnerabilitySubKPIs(vulnerabilitiesWithDependencies, maximalDependencyVulnerabilityKPI)
-// securityKPI.addChildKPI(maximalDependencyVulnerabilityKPI, 0.5)
-//
-// // TODO: There might be the case that there are no latest tool runs for this repository (would end up in an exception)
-// val latestToolRunOCCMD = toolRunService.getLatestToolRunsForRepo(repoId).filter { it.tool?.name == "OCCMD" }[0]
-// val secretKpi = KPICreateDto(
-// "Public Secrets",
-// "Score to indicate whether the repository contains secrets like passwords or api keys.",
-// false,
-// RawValueKPICalculationStrategy(if (latestToolRunOCCMD.findings.isEmpty()) 100 else 0)
-// )
-
-// securityKPI.addChildKPI(secretKpi, 0.5)
+ var dependencyWeight = 1.0
+ if (occmdKpis.secretsKpi != null) {
+ dependencyWeight -= 0.3
+ securityKPI.addChildKPI(occmdKpis.secretsKpi, 0.3)
+ }
+ if (occmdKpis.checkedInBinaries != null) {
+ dependencyWeight -= 0.2
+ securityKPI.addChildKPI(occmdKpis.checkedInBinaries, 0.2)
+ }
+ securityKPI.addChildKPI(maximalDependencyVulnerabilityKPI, dependencyWeight)
+
val rootKPI = KPICreateDto(
"Project Score",
"Assesses the project resp. the provided software in the aspects of maturity (based on quality, security and usability aspects) as well as development process.",
@@ -119,42 +117,6 @@ class KPIService(
return rootKPI
}
-// private fun addMaximalDependencyVulnerabilitySubKPIs(
-// vulnerabilitiesWithDependencies: HashMap<VulnerabilityDto, MutableSet<String>>,
-// maximalDependencyVulnerabilityKPI: KPICreateDto
-// ) {
-// // We iterate through the vulnerabilities and create a sub (raw data) kpi for each of them. Dependencies are displayed in the description
-// for ((vulnerability, dependencies) in vulnerabilitiesWithDependencies) {
-// maximalDependencyVulnerabilityKPI.addChildKPI(
-// KPICreateDto(
-// "Vulnerability " + vulnerability.cveIdentifier,
-// "Affected packages: " + dependencies.joinToString(", "),
-// false,
-// // TODO: We multiply the CVSS score by 10 as the data structure can only hold integer values. This might be changed in the future.
-// RawValueKPICalculationStrategy((vulnerability.vulnerabilityScores.first().severity.toDouble() * 10).toInt()),
-// // TODO: Displayscore shows the real CVSS value, might not be needed if above TODO is resolved
-// displayScore = vulnerability.vulnerabilityScores.first().severity
-// ), 0.0
-// )
-// }
-// }
-//
-// private fun restructureDependenciesAndVulnerabilities(dependencyDtos: List<DependencyDto>): HashMap<VulnerabilityDto, MutableSet<String>> {
-// val vulnerabilitiesWithDependencies = HashMap<VulnerabilityDto, MutableSet<String>>()
-// // Switch structure around, so we can see which vulnerabilities are there and which dependencies are associated to them
-// for (dependencyDto in dependencyDtos) {
-// // Iterate through the list of vulnerabilities associated with each dependency
-// for (vulnerability in dependencyDto.vulnerabilities) {
-// // Check if the vulnerability is already present in the result map
-// // If not, add it with an empty list of dependencies
-// vulnerabilitiesWithDependencies.computeIfAbsent(vulnerability) { mutableSetOf() }
-// // Add the current dependency to the list of dependencies for the vulnerability
-// vulnerabilitiesWithDependencies[vulnerability]?.add(dependencyDto.name)
-// }
-// }
-// return vulnerabilitiesWithDependencies
-// }
-
private fun calculateKPIsRecursively(kpi: KPICreateDto, visited: MutableSet<KPICreateDto>) {
// Check if the KPI has already been visited
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/OccmdTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/OccmdTask.kt
index aa15fca3..1fae49f9 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/OccmdTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/OccmdTask.kt
@@ -3,8 +3,8 @@ package de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd
import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService
import de.fraunhofer.iem.dataprovider.taskManager.events.Event
import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ToolProcessTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.enumeration.Checks
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.json.raw.RawResultJson
+import de.fraunhofer.iem.dataprovider.tools.occmd.enumeration.Checks
+import de.fraunhofer.iem.dataprovider.tools.occmd.json.RawResultJson
import kotlinx.serialization.json.Json
import java.nio.file.Path
import java.util.*
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/sarif/OccmdSarif.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/sarif/OccmdSarif.kt
deleted file mode 100644
index 0c263fd0..00000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/sarif/OccmdSarif.kt
+++ /dev/null
@@ -1,46 +0,0 @@
-package de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.json.sarif
-//
-//import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.json.raw.OccmdRaw
-//import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.sarif.json.*
-//import kotlinx.serialization.json.Json
-//import java.io.IOException
-//import java.nio.file.Path
-//
-//
-//fun getOccmdSarifFromFilePath(resultPath: Path): SarifJson {
-// val resFile = resultPath.toFile()
-// if (resFile.exists()) {
-// val resString = resFile.readText()
-// val json = Json { ignoreUnknownKeys = true}
-// val occmdRaw = json.decodeFromString<OccmdRaw>(resString)
-//
-// return occmdRawToSarif(occmdRaw)
-// }
-//
-// throw IOException("File not found / Path is no file.")
-//}
-//
-//private fun occmdRawToSarif(occmdRaw: OccmdRaw): SarifJson {
-// val tool = SarifToolJson(SarifDriverJson(name = "OCCMD", fullName = "OCCMD", version = occmdRaw.version))
-//
-// val results: MutableList<SarifResultJson> = mutableListOf()
-//
-// for (elem in occmdRaw.results) {
-// for (rawResult in elem.value) {
-// val location = SarifLocationJson(
-// SarifPhysicalLocationJson(
-// SarifArtifactLocationJson(rawResult.filename),
-// SarifRegionJson(startLine = rawResult.line_number)
-// )
-// )
-//
-// val message = SarifMessageJson("hashed_string: ${rawResult.hashed_secret} and is_verified: ${rawResult.is_verified}")
-// val result = SarifResultJson("warning", listOf(location), message, rawResult.type)
-//
-// results.add(result)
-// }
-// }
-//
-// val run = SarifRunJson(tool = tool, results = results)
-// return SarifJson(listOf(run))
-//}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/dto/OccmdKpiDto.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/dto/OccmdKpiDto.kt
new file mode 100644
index 00000000..69ca4e62
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/dto/OccmdKpiDto.kt
@@ -0,0 +1,9 @@
+package de.fraunhofer.iem.dataprovider.tools.occmd.dto
+
+import de.fraunhofer.iem.dataprovider.kpi.dto.KPICreateDto
+
+data class OccmdKpisDto(
+ val secretsKpi: KPICreateDto?,
+ val checkedInBinaries: KPICreateDto?,
+ val sastUsage: KPICreateDto?
+)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/entity/OccmdSecretEntity.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/entity/OccmdSecretEntity.kt
deleted file mode 100644
index 20b2dcae..00000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/entity/OccmdSecretEntity.kt
+++ /dev/null
@@ -1,33 +0,0 @@
-package de.fraunhofer.iem.dataprovider.tools.occmd.entity
-
-import jakarta.persistence.*
-import org.hibernate.Hibernate
-import java.util.*
-
-@Entity
-@Table(name = "occmdsecret")
-class OccmdSecretEntity {
-
- @Id
- @GeneratedValue(strategy = GenerationType.AUTO)
- @Column(name = "id", nullable = false)
- var id: UUID? = null
-
- @Column(name = "score")
- var score: Double? = null
-
- @ElementCollection(targetClass = String::class, fetch = FetchType.EAGER)
- @CollectionTable(name = "occmdsecret_results", joinColumns = [JoinColumn(name = "occmdsecret_id")])
- @Column(name = "results", length = 512, nullable = false)
- var results: MutableList<String> = mutableListOf()
-
- override fun equals(other: Any?): Boolean {
- if (this === other) return true
- if (other == null || Hibernate.getClass(this) != Hibernate.getClass(other)) return false
- other as OccmdSecretEntity
-
- return id != null && id == other.id
- }
-
- override fun hashCode(): Int = javaClass.hashCode()
-}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/enumeration/Checks.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/enumeration/Checks.kt
similarity index 79%
rename from src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/enumeration/Checks.kt
rename to src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/enumeration/Checks.kt
index 1af0b112..635151c2 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/enumeration/Checks.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/enumeration/Checks.kt
@@ -1,4 +1,4 @@
-package de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.enumeration
+package de.fraunhofer.iem.dataprovider.tools.occmd.enumeration
enum class Checks(val checkName: String) {
SastUsageBasic("SastUsageBasic"), Secrets("Secrets"), CheckedInBinaries("CheckedInBinaries");
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/RawResultJson.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/RawResultJson.kt
similarity index 84%
rename from src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/RawResultJson.kt
rename to src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/RawResultJson.kt
index 80b1280c..42bedf02 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/RawResultJson.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/RawResultJson.kt
@@ -1,4 +1,4 @@
-package de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.json.raw
+package de.fraunhofer.iem.dataprovider.tools.occmd.json
import kotlinx.serialization.SerialName
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/SecretResultsJson.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/SecretResultsJson.kt
similarity index 73%
rename from src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/SecretResultsJson.kt
rename to src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/SecretResultsJson.kt
index d2866235..552bd343 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/SecretResultsJson.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/SecretResultsJson.kt
@@ -1,4 +1,4 @@
-package de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.json.raw
+package de.fraunhofer.iem.dataprovider.tools.occmd.json
import kotlinx.serialization.SerialName
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/ToolSecretJson.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/ToolSecretJson.kt
similarity index 76%
rename from src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/ToolSecretJson.kt
rename to src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/ToolSecretJson.kt
index 721f7d0a..2d4121dd 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/ToolSecretJson.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/ToolSecretJson.kt
@@ -1,4 +1,4 @@
-package de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.json.raw
+package de.fraunhofer.iem.dataprovider.tools.occmd.json
import kotlinx.serialization.SerialName
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/ToolSecretResultJson.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/ToolSecretResultJson.kt
similarity index 76%
rename from src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/ToolSecretResultJson.kt
rename to src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/ToolSecretResultJson.kt
index 45b6e44c..d803d1cd 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/tools/occmd/json/raw/ToolSecretResultJson.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/json/ToolSecretResultJson.kt
@@ -1,4 +1,4 @@
-package de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.occmd.json.raw
+package de.fraunhofer.iem.dataprovider.tools.occmd.json
import kotlinx.serialization.SerialName
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt
new file mode 100644
index 00000000..e6cd93aa
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt
@@ -0,0 +1,131 @@
+package de.fraunhofer.iem.dataprovider.tools.occmd.service
+
+import de.fraunhofer.iem.dataprovider.configuration.DirectoryPathsProperties
+import de.fraunhofer.iem.dataprovider.configuration.OpenCodeGitlabApiProperties
+import de.fraunhofer.iem.dataprovider.kpi.dto.KPICreateDto
+import de.fraunhofer.iem.dataprovider.kpi.strategy.RawValueKPICalculationStrategy
+import de.fraunhofer.iem.dataprovider.logger.getLogger
+import de.fraunhofer.iem.dataprovider.tools.occmd.dto.OccmdKpisDto
+import de.fraunhofer.iem.dataprovider.tools.occmd.enumeration.Checks
+import de.fraunhofer.iem.dataprovider.tools.occmd.json.RawResultJson
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.future.await
+import kotlinx.coroutines.withContext
+import kotlinx.serialization.json.Json
+import org.eclipse.jgit.api.Git
+import org.springframework.stereotype.Service
+import java.io.File
+import java.nio.file.Files.isExecutable
+import java.nio.file.Paths
+import java.util.*
+import kotlin.io.path.ExperimentalPathApi
+import kotlin.io.path.deleteRecursively
+
+@Service
+class OccmdService(
+ private val dirProperties: DirectoryPathsProperties,
+ private val gitlabApiProperties: OpenCodeGitlabApiProperties
+) {
+
+ val logger = getLogger(javaClass)
+
+ @OptIn(ExperimentalPathApi::class)
+ suspend fun runOccmd(repoId: Long, repoUrl: String): List<RawResultJson> {
+ // clone repo
+ val outDir = Paths.get(dirProperties.gitCloneTargetDirectory, "$repoId-${Date().time}")
+ cloneGit(repoUrl, outDir.toFile())
+ // run tool
+ val rawOccmdResults = runOccmd(
+ dirProperties.occmdPath, arrayOf(
+ outDir.toString(), "$repoId",
+ gitlabApiProperties.accessToken
+ )
+ )
+ // TODO: right now we fire and forget, in a sense that we run the tool calculate the
+ // KPIs and forget the tool results. For this tool we want to manually store the results
+ // until it is included into the official CI/CD pipeline.
+
+ // delete cloned repo
+ try {
+ outDir.deleteRecursively()
+ } catch (e: Exception) {
+ logger.error("Delete directory after occmd run failed for dir $outDir")
+ }
+
+ return rawOccmdResults
+ }
+
+ fun calculateOccmdKpis(rawOccmdResults: List<RawResultJson>): OccmdKpisDto {
+ var secretsDto: KPICreateDto? = null
+ var checkedInBinaryDto: KPICreateDto? = null
+ var sastUsageDto: KPICreateDto? = null
+
+ rawOccmdResults.forEach {
+ val check = Checks.fromString(it.check)
+ when (check) {
+ Checks.CheckedInBinaries ->
+ checkedInBinaryDto = KPICreateDto(
+ "Checked in Binary",
+ "",
+ false,
+ RawValueKPICalculationStrategy((it.score * 10).toInt())
+ )
+
+ Checks.SastUsageBasic ->
+ sastUsageDto = KPICreateDto(
+ "SAST usage",
+ "",
+ false,
+ RawValueKPICalculationStrategy((it.score * 10).toInt())
+ )
+
+ Checks.Secrets ->
+ secretsDto = KPICreateDto(
+ "Secrets",
+ "",
+ false,
+ RawValueKPICalculationStrategy((it.score * 10).toInt())
+ )
+
+ else ->
+ logger.warn("Unknown result")
+ }
+ }
+
+ return OccmdKpisDto(secretsKpi = secretsDto, checkedInBinaries = checkedInBinaryDto, sastUsage = sastUsageDto)
+ }
+
+ private suspend fun runOccmd(execPath: String, flags: Array<String>): List<RawResultJson> = coroutineScope {
+ val toolResults = mutableListOf<RawResultJson>()
+ if (isExecutable(execPath)) {
+
+ val process = withContext(Dispatchers.IO) {
+ ProcessBuilder(execPath, *flags).start()
+ }
+ process.onExit().await()
+ val json = Json { ignoreUnknownKeys = true }
+ process.inputStream.bufferedReader().forEachLine {
+ toolResults.add(json.decodeFromString(it))
+ }
+ } else {
+ logger.warn("Given execPath is not an executable $execPath.")
+ // TODO: we should probably throw an exception here.
+ }
+
+ return@coroutineScope toolResults
+ }
+
+ private fun isExecutable(filePath: String): Boolean {
+ val path = Paths.get(filePath)
+ return isExecutable(path)
+ }
+
+ private suspend fun cloneGit(repoUrl: String, outDir: File) {
+ val git: Git = Git.cloneRepository()
+ .setURI(repoUrl)
+ .setDirectory(outDir)
+ .call()
+ git.close()
+ }
+}
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index 7fbe091c..5a877d2a 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -3,19 +3,23 @@ spring.config.import=optional:classpath:.env[.properties]
# Token can be an empty string to access public repositories only
opencode.host=https://gitlab.opencode.de/
opencode.access-token=${OC_GL_APIKEY}
+
# Tool APIs
opencode.api.base-path=https://sl.dev.o4oe.de/api/v1/project/
opencode.api.ort=/cve-result
# DB Login data
-spring.datasource.url=jdbc:postgresql://${host}:26257/${DB_USER}?sslmode=${ssl_mode}&sslrootcert=${ca_crt}&sslcert=${ssl_cert}&sslkey=${ssl_key}
+spring.datasource.url=${DB_URL}
+#spring.datasource.url=jdbc:postgresql://${host}:26257/${DB_USER}?sslmode=${ssl_mode}&sslrootcert=${ca_crt}&sslcert=${ssl_cert}&sslkey=${ssl_key}
spring.datasource.username=${DB_USER}
spring.datasource.password=${DB_PW}
# API key to access this server's API
security.api-key=${API_KEY}
security.admin-password=${ADMIN_PASSWORD}
-directories.git-clone-target-directory=${GIT_CLONE_TARGET_DIRECTORY}
-directories.tool-results-target-directory=${TOOL_RESULTS_TARGET_DIRECTORY}
+occmd.git-clone-target-directory=${GIT_CLONE_TARGET_DIRECTORY}
+occmd.tool-results-target-directory=${TOOL_RESULTS_TARGET_DIRECTORY}
+occmd.occmd-path=${OCCMD_PATH}
+
server.port=${PORT}
# Generates db schema if it doesn't exist in db
spring.jpa.generate-ddl=true
diff --git a/src/main/resources/scripts/occmd.sh b/src/main/resources/scripts/occmd.sh
index 0ca1d547..bcd208ef 100755
--- a/src/main/resources/scripts/occmd.sh
+++ b/src/main/resources/scripts/occmd.sh
@@ -2,8 +2,8 @@
INSTALL_DIR=/occmd
PROJ_PATH=${1}
PROJ_ID=${2}
-OUT_FILE_PATH=${3}
+API_KEY=${3}
-#.${INSTALL_DIR}/occmd check -d "${PROJ_PATH}" -i "${PROJ_ID}"
-(docker run -e OC_GL_APIKEY="${OC_GL_APIKEY}" -v "${PROJ_PATH}":/opt/PROJ --rm occmd check -d /opt/PROJ -i "${PROJ_ID}") > "${OUT_FILE_PATH}"
+#export OC_GL_APIKEY="${API_KEY}" && .${INSTALL_DIR}/occmd check -d "${PROJ_PATH}" -i "${PROJ_ID}"
+(docker run -e OC_GL_APIKEY="${API_KEY}" -v "${PROJ_PATH}":/opt/PROJ --rm occmd check -d /opt/PROJ -i "${PROJ_ID}")
exit 0
\ No newline at end of file
diff --git a/src/main/resources/scripts/ort/config.yml b/src/main/resources/scripts/ort/config.yml
deleted file mode 100644
index 824c3f61..00000000
--- a/src/main/resources/scripts/ort/config.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-ort:
- advisor:
- osv:
- serverUrl: "https://api-staging.osv.dev"
- forceOverwrite: true
\ No newline at end of file
diff --git a/src/main/resources/scripts/ort/ort_advisor.sh b/src/main/resources/scripts/ort/ort_advisor.sh
deleted file mode 100755
index db8103bb..00000000
--- a/src/main/resources/scripts/ort/ort_advisor.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-
-PROJECT_DIRECTORY=${1}
-OUTPUT_DIRECTORY=${2}
-echo Advisor Started
-echo Input directory: "${PROJECT_DIRECTORY}" Output directory: "${OUTPUT_DIRECTORY}"
-
-script_dir="$(dirname "$0")"
-
- #add --network vulnerablecode_mynetwork when running together with a custom vulnerable code instance
-docker run --rm \
--v "${script_dir}":/config \
--v "${PROJECT_DIRECTORY}":/project \
--v "${OUTPUT_DIRECTORY}":/result \
-ort --config /config/config.yml --info advise -f JSON -i /project/analyzer-result.json --output-dir /result -a OSV
-exit 0
\ No newline at end of file
diff --git a/src/main/resources/scripts/ort/ort_analyzer.sh b/src/main/resources/scripts/ort/ort_analyzer.sh
deleted file mode 100755
index c2a5d860..00000000
--- a/src/main/resources/scripts/ort/ort_analyzer.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-
-PROJECT_DIRECTORY=${1}
-OUTPUT_DIRECTORY=${2}
-echo Analyzer started
-echo Input directory: "${PROJECT_DIRECTORY}" Output directory: "${OUTPUT_DIRECTORY}"
-
-script_dir="$(dirname "$0")"
-
-docker run --rm \
--v "${script_dir}":/config \
--v "${PROJECT_DIRECTORY}":/project \
--v "${OUTPUT_DIRECTORY}":/result \
-ort --info --config /config/config.yml analyze -i /project --output-dir /result -f JSON
-
-exit 0
\ No newline at end of file
--
GitLab