From 38d59c8961416c764ac0dccebcbab45cf2be3b99 Mon Sep 17 00:00:00 2001
From: Jan-Niclas Struewer <j.n.struewer@gmail.com>
Date: Wed, 27 Sep 2023 14:45:52 +0200
Subject: [PATCH] create tool db objects upon application start iff they don't
exist
---
.../configuration/DirectoryPathsProperties.kt | 4 ----
.../dataprovider/kpi/metrics/MetricsService.kt | 13 ++++++++++++-
.../service/RepositoryDetailsService.kt | 11 ++++++++++-
.../iem/dataprovider/tool/entity/ToolEntity.kt | 4 ----
.../iem/dataprovider/tool/service/ToolService.kt | 4 ----
.../toolRun/service/ToolRunService.kt | 4 ++--
.../tools/occmd/service/OccmdService.kt | 16 +++++++++++++---
.../dataprovider/tools/ort/service/OrtService.kt | 12 +++++++++++-
src/main/resources/application.properties | 13 ++++++-------
9 files changed, 54 insertions(+), 27 deletions(-)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/configuration/DirectoryPathsProperties.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/configuration/DirectoryPathsProperties.kt
index 951661e2..93e693c5 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/configuration/DirectoryPathsProperties.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/configuration/DirectoryPathsProperties.kt
@@ -19,15 +19,11 @@ data class DirectoryPathsProperties(
@NotBlank
@Length(min = 1)
val gitCloneTargetDirectory: String,
- @NotBlank
- @Length(min = 1)
- val toolResultsTargetDirectory: String
) {
@PostConstruct
fun postConstruct() {
createDir(gitCloneTargetDirectory)
- createDir(toolResultsTargetDirectory)
}
private fun createDir(stringPath: String) {
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt
index d3a99773..53c68767 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt
@@ -25,6 +25,7 @@ class MetricsService(
private val defaultScope = CoroutineScope(Dispatchers.Default)
val ioScope = CoroutineScope(Dispatchers.IO)
+
suspend fun handleRepositoryChange(repoId: Long) = defaultScope.launch {
// TODO: we need to create the tools in the db and link them to the tool run.
@@ -66,16 +67,24 @@ class MetricsService(
val vulnerabilityKpis = async {
val vulnerabilityDtos = ortService.getOrtResults(repoId) // in the dev setup we get results for repo id 106
- ortService.calculateVulnerabilityKpis(vulnerabilityDtos)
+ val kpis = ortService.calculateVulnerabilityKpis(vulnerabilityDtos)
+ if (kpis.isNotEmpty()) {
+ toolRun.toolEntities.add(ortService.toolEntity)
+ }
+ kpis
}
val repositoryDetailsKpi = async {
val repoDetailsDto = repositoryDetailsService.getRepositoryDetails(repoId)
+ toolRun.toolEntities.add(repositoryDetailsService.toolEntity)
repositoryDetailsService.calculateRepositoryDetailsKpis(repoDetailsDto)
}
val occmdKpis = async {
val rawOccmdResults = occmdService.runOccmd(repoId, toolRun.repository?.url!!)
+ if (rawOccmdResults.isNotEmpty()) {
+ toolRun.toolEntities.add(occmdService.toolEntity)
+ }
occmdService.calculateOccmdKpis(rawOccmdResults)
}
@@ -90,5 +99,7 @@ class MetricsService(
vulnerabilityKpis = vulnerabilityKpis.await(),
occmdKpis = occmdKpis.await()
)
+
+ toolRunService.saveToolRunEntitiy(toolRun)
}
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/service/RepositoryDetailsService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/service/RepositoryDetailsService.kt
index ab53011e..9e1821b8 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/service/RepositoryDetailsService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/service/RepositoryDetailsService.kt
@@ -5,15 +5,24 @@ import de.fraunhofer.iem.dataprovider.kpi.dto.KPICreateDto
import de.fraunhofer.iem.dataprovider.kpi.metrics.repositoryDetails.dto.RepositoryDetailsKpisDto
import de.fraunhofer.iem.dataprovider.kpi.strategy.RawValueKPICalculationStrategy
import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryDetailsDto
+import de.fraunhofer.iem.dataprovider.tool.dto.CreateToolDto
+import de.fraunhofer.iem.dataprovider.tool.entity.ToolEntity
+import de.fraunhofer.iem.dataprovider.tool.service.ToolService
import org.springframework.stereotype.Service
@Service
-class RepositoryDetailsService(private val openCodeGitlabApi: OpenCodeGitlabApi) {
+class RepositoryDetailsService(private val openCodeGitlabApi: OpenCodeGitlabApi, private val toolService: ToolService) {
suspend fun getRepositoryDetails(repoId: Long): RepositoryDetailsDto {
return openCodeGitlabApi.getRepositoryDetails(repoId)
}
+ val toolEntity: ToolEntity = getOrCreateToolEntity()
+
+ private final fun getOrCreateToolEntity(): ToolEntity {
+ val createToolDto = CreateToolDto("Gitlab API")
+ return toolService.findOrCreateTool(createToolDto)
+ }
/**
* Creates a named map of RepositoryCreateDtos, based upon the provided repository details.
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tool/entity/ToolEntity.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tool/entity/ToolEntity.kt
index c9329074..7ba9a106 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tool/entity/ToolEntity.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tool/entity/ToolEntity.kt
@@ -19,10 +19,6 @@ class ToolEntity {
@Column(name = "name", length = 500)
var name: String? = null
- @Column(name = "api")
- var api: String? = null
-
-
@CurrentTimestamp(event = [EventType.INSERT])
var createdAt: Instant? = null
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tool/service/ToolService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tool/service/ToolService.kt
index 05bb683e..a0285d0e 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tool/service/ToolService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tool/service/ToolService.kt
@@ -10,10 +10,6 @@ class ToolService(
private val toolRepository: ToolRepository,
) {
- fun save(tool: ToolEntity) {
- toolRepository.save(tool)
- }
-
fun findOrCreateTool(tool: CreateToolDto): ToolEntity {
return toolRepository.findByNameIgnoreCase(
tool.name
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt
index f9c5b430..b9b34d27 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt
@@ -17,7 +17,7 @@ class ToolRunService(private val toolRunRepository: ToolRunRepository) {
return toolRunRepository.save(tr)
}
- fun getLatestToolResultsForRepo(repo: RepositoryEntity): List<ToolRunEntity> {
- return repo.toolRuns
+ fun saveToolRunEntitiy(toolRunEntity: ToolRunEntity): ToolRunEntity {
+ return toolRunRepository.save(toolRunEntity)
}
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt
index cd5c9c87..3fd1e480 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt
@@ -5,6 +5,9 @@ import de.fraunhofer.iem.dataprovider.configuration.OpenCodeGitlabApiProperties
import de.fraunhofer.iem.dataprovider.kpi.dto.KPICreateDto
import de.fraunhofer.iem.dataprovider.kpi.strategy.RawValueKPICalculationStrategy
import de.fraunhofer.iem.dataprovider.logger.getLogger
+import de.fraunhofer.iem.dataprovider.tool.dto.CreateToolDto
+import de.fraunhofer.iem.dataprovider.tool.entity.ToolEntity
+import de.fraunhofer.iem.dataprovider.tool.service.ToolService
import de.fraunhofer.iem.dataprovider.tools.occmd.dto.OccmdKpisDto
import de.fraunhofer.iem.dataprovider.tools.occmd.enumeration.Checks
import de.fraunhofer.iem.dataprovider.tools.occmd.json.RawResultJson
@@ -25,11 +28,19 @@ import kotlin.io.path.deleteRecursively
@Service
class OccmdService(
private val dirProperties: DirectoryPathsProperties,
- private val gitlabApiProperties: OpenCodeGitlabApiProperties
+ private val gitlabApiProperties: OpenCodeGitlabApiProperties,
+ private val toolService: ToolService
) {
val logger = getLogger(javaClass)
+ val toolEntity: ToolEntity = getOrCreateToolEntity()
+
+ private final fun getOrCreateToolEntity(): ToolEntity {
+ val createToolDto = CreateToolDto("OCCMD")
+ return toolService.findOrCreateTool(createToolDto)
+ }
+
@OptIn(ExperimentalPathApi::class)
suspend fun runOccmd(repoId: Long, repoUrl: String): List<RawResultJson> {
// clone repo
@@ -109,8 +120,7 @@ class OccmdService(
toolResults.add(json.decodeFromString(it))
}
} else {
- logger.warn("Given execPath is not an executable $execPath.")
- // TODO: we should probably throw an exception here.
+ logger.error("Given execPath is not an executable $execPath.")
}
return@coroutineScope toolResults
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/ort/service/OrtService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/ort/service/OrtService.kt
index 1b446dd5..24bcc0eb 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/ort/service/OrtService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/ort/service/OrtService.kt
@@ -3,6 +3,9 @@ package de.fraunhofer.iem.dataprovider.tools.ort.service
import de.fraunhofer.iem.dataprovider.configuration.OpenCodeApiProperties
import de.fraunhofer.iem.dataprovider.kpi.dto.KPICreateDto
import de.fraunhofer.iem.dataprovider.kpi.strategy.RawValueKPICalculationStrategy
+import de.fraunhofer.iem.dataprovider.tool.dto.CreateToolDto
+import de.fraunhofer.iem.dataprovider.tool.entity.ToolEntity
+import de.fraunhofer.iem.dataprovider.tool.service.ToolService
import de.fraunhofer.iem.dataprovider.tools.ort.dto.VulnerabilityDto
import de.fraunhofer.iem.dataprovider.tools.ort.json.OrtJson
import io.ktor.client.*
@@ -16,7 +19,7 @@ import kotlinx.serialization.json.Json
import org.springframework.stereotype.Service
@Service
-class OrtService(private val openCodeApiProperties: OpenCodeApiProperties) {
+class OrtService(private val openCodeApiProperties: OpenCodeApiProperties, private val toolService: ToolService) {
val client = HttpClient(CIO) {
install(ContentNegotiation) {
@@ -26,6 +29,13 @@ class OrtService(private val openCodeApiProperties: OpenCodeApiProperties) {
}
}
+ val toolEntity: ToolEntity = getOrCreateToolEntity()
+
+ private final fun getOrCreateToolEntity(): ToolEntity {
+ val createToolDto = CreateToolDto("ORT")
+ return toolService.findOrCreateTool(createToolDto)
+ }
+
suspend fun getOrtResults(repoId: Long): List<VulnerabilityDto> {
val response: HttpResponse = client.get(getToolApiPath(repoId))
val ortJson = response.body<OrtJson>()
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index 5a877d2a..d31e4dbc 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -8,19 +8,14 @@ opencode.access-token=${OC_GL_APIKEY}
opencode.api.base-path=https://sl.dev.o4oe.de/api/v1/project/
opencode.api.ort=/cve-result
-# DB Login data
-spring.datasource.url=${DB_URL}
-#spring.datasource.url=jdbc:postgresql://${host}:26257/${DB_USER}?sslmode=${ssl_mode}&sslrootcert=${ca_crt}&sslcert=${ssl_cert}&sslkey=${ssl_key}
-spring.datasource.username=${DB_USER}
-spring.datasource.password=${DB_PW}
# API key to access this server's API
security.api-key=${API_KEY}
security.admin-password=${ADMIN_PASSWORD}
occmd.git-clone-target-directory=${GIT_CLONE_TARGET_DIRECTORY}
-occmd.tool-results-target-directory=${TOOL_RESULTS_TARGET_DIRECTORY}
occmd.occmd-path=${OCCMD_PATH}
server.port=${PORT}
+
# Generates db schema if it doesn't exist in db
spring.jpa.generate-ddl=true
spring.jpa.show-sql=false
@@ -31,4 +26,8 @@ spring.jpa.properties.hibernate.bytecode.use_reflection_optimizer=false
spring.jpa.open-in-view=false
spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false
spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.CockroachDialect
-#logging.level.root=DEBUG
\ No newline at end of file
+# DB Login data
+spring.datasource.url=${DB_URL}
+#spring.datasource.url=jdbc:postgresql://${host}:26257/${DB_USER}?sslmode=${ssl_mode}&sslrootcert=${ca_crt}&sslcert=${ssl_cert}&sslkey=${ssl_key}
+spring.datasource.username=${DB_USER}
+spring.datasource.password=${DB_PW}
\ No newline at end of file
--
GitLab