From a8b744c11c7cec8df40aa0220c6d74e11a3e7590 Mon Sep 17 00:00:00 2001 From: Jan-Niclas Struewer <j.n.struewer@gmail.com> Date: Wed, 13 Dec 2023 14:07:22 +0100 Subject: [PATCH] updated database behavior to fix transactional problems --- .../dataprovider/kpi/service/KPIService.kt | 14 +--- .../controller/RepositoryController.kt | 7 +- .../repository/service/RepositoryService.kt | 14 +--- .../toolRun/entity/ToolRunEntity.kt | 4 +- .../toolRun/repository/ToolRunRepository.kt | 2 +- .../toolRun/service/ToolRunDbService.kt | 62 +++++++++++++++ .../toolRun/service/ToolRunService.kt | 78 ++++++------------- .../service/RepositoryDetailsService.kt | 8 +- .../tools/occmd/service/OccmdService.kt | 11 +-- .../tools/ort/service/OrtService.kt | 9 +-- 10 files changed, 97 insertions(+), 112 deletions(-) create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunDbService.kt diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt index 4127ba4a..6aaac16a 100644 --- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt @@ -4,24 +4,19 @@ import de.fraunhofer.iem.dataprovider.kpi.dto.KPITreeChildResponseDto import de.fraunhofer.iem.dataprovider.kpi.dto.KPITreeResponseDto import de.fraunhofer.iem.dataprovider.kpi.dto.KpiCalculationDto import de.fraunhofer.iem.dataprovider.kpi.dto.RawValueKpiCreateDto -import de.fraunhofer.iem.dataprovider.kpi.entity.KPIEntity import de.fraunhofer.iem.dataprovider.kpi.enumeration.KpiKind -import de.fraunhofer.iem.dataprovider.kpi.repository.KPIRepository import de.fraunhofer.iem.dataprovider.kpi.strategy.AggregationKPICalculationStrategy import de.fraunhofer.iem.dataprovider.kpi.strategy.MaximumKPICalculationStrategy import de.fraunhofer.iem.dataprovider.kpi.strategy.RatioKPICalculationStrategy import de.fraunhofer.iem.dataprovider.logger.getLogger import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryDetailsDto -import de.fraunhofer.iem.dataprovider.toolRun.entity.ToolRunEntity import de.fraunhofer.iem.dataprovider.tools.occmd.enumeration.Checks import de.fraunhofer.iem.dataprovider.tools.occmd.json.RawResultJson import de.fraunhofer.iem.dataprovider.tools.ort.dto.VulnerabilityDto import org.springframework.stereotype.Service @Service -class KPIService( - private val kpiRepository: KPIRepository, -) { +class KPIService { private val logger = getLogger(javaClass) @@ -141,13 +136,6 @@ class KPIService( ) } - fun saveRawKpis( - toolRunEntity: ToolRunEntity, - rawKpis: List<RawValueKpiCreateDto> - ): List<KPIEntity> { - return kpiRepository.saveAll(rawKpis.map { it.toDbObject(toolRunEntity) }) - } - fun getKpiTreeForRawKpis(rawKpis: List<KpiCalculationDto>): KpiCalculationDto { val rawValueKpis = rawKpis.toMutableList() val vulnerabilities = rawValueKpis.filter { it.kind == KpiKind.VULNERABILITY_SCORE } diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/controller/RepositoryController.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/controller/RepositoryController.kt index b0e2bf64..04190660 100644 --- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/controller/RepositoryController.kt +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/controller/RepositoryController.kt @@ -148,15 +148,10 @@ class RepositoryController( false } - logger.info("Get repository with id $id") - val repositoryEntity = this.repositoryService.findRepoById(id) ?: throw ResponseStatusException( - HttpStatus.NOT_FOUND, "repository not found" - ) - return ToolRunResponseDto( isProjectMember = isProjectMember, toolRun = toolRunService.getToolRunForRepository( - repo = repositoryEntity, + projectId = id, includeFindings = isProjectMember ) ) diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/service/RepositoryService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/service/RepositoryService.kt index 16f1dd23..2d1bc0b0 100644 --- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/service/RepositoryService.kt +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/service/RepositoryService.kt @@ -7,14 +7,12 @@ import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryCreateDto import de.fraunhofer.iem.dataprovider.repository.entity.RepositoryEntity import de.fraunhofer.iem.dataprovider.repository.repository.RepositoryRepository import org.springframework.stereotype.Service -import java.util.* @Service class RepositoryService( private val repositoryRepository: RepositoryRepository, private val openCodeGitlabApi: OpenCodeGitlabApi, - - ) { +) { private val logger = getLogger(javaClass) @@ -35,18 +33,10 @@ class RepositoryService( /** * Queries the gitlab api to get the repo name, url, and id. */ - fun getRepositoryInfo(projectId: Long): RepositoryCreateDto { + fun getRepositoryInfoFromGitlab(projectId: Long): RepositoryCreateDto { return openCodeGitlabApi.getRepositoryInfo(projectId) } - fun findRepoByID(id: UUID): RepositoryEntity? { - val repository = repositoryRepository.findById(id) - if (repository.isEmpty) { - return null - } - return repository.get() - } - fun findRepoById(projectId: Long): RepositoryEntity? { return repositoryRepository.findByProjectId(projectId) } diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/entity/ToolRunEntity.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/entity/ToolRunEntity.kt index ff81e910..38fffeee 100644 --- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/entity/ToolRunEntity.kt +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/entity/ToolRunEntity.kt @@ -29,10 +29,10 @@ class ToolRunEntity( ) val toolEntities: MutableSet<ToolEntity> = mutableSetOf(), - @OneToMany(mappedBy = "toolRunEntity", orphanRemoval = true) + @OneToMany(cascade = [CascadeType.ALL], mappedBy = "toolRunEntity", orphanRemoval = true) val kpiEntities: MutableSet<KPIEntity> = mutableSetOf(), - @OneToMany(mappedBy = "toolRunEntity", orphanRemoval = true) + @OneToMany(cascade = [CascadeType.ALL], mappedBy = "toolRunEntity", orphanRemoval = true) val languageEntities: MutableSet<LanguageEntity> = mutableSetOf(), @CurrentTimestamp(event = [EventType.INSERT]) diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/repository/ToolRunRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/repository/ToolRunRepository.kt index bafffc17..1fb64ba2 100644 --- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/repository/ToolRunRepository.kt +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/repository/ToolRunRepository.kt @@ -5,5 +5,5 @@ import org.springframework.data.jpa.repository.JpaRepository import java.util.* interface ToolRunRepository : JpaRepository<ToolRunEntity, UUID> { - fun findFirstByRepository_IdOrderByCreatedAtDesc(id: UUID): ToolRunEntity + fun findFirstByRepository_ProjectIdOrderByCreatedAtDesc(projectId: Long): ToolRunEntity } diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunDbService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunDbService.kt new file mode 100644 index 00000000..6251d5b9 --- /dev/null +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunDbService.kt @@ -0,0 +1,62 @@ +package de.fraunhofer.iem.dataprovider.toolRun.service + +import de.fraunhofer.iem.dataprovider.kpi.dto.RawValueKpiCreateDto +import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryCreateDto +import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService +import de.fraunhofer.iem.dataprovider.tool.dto.CreateToolDto +import de.fraunhofer.iem.dataprovider.tool.service.ToolService +import de.fraunhofer.iem.dataprovider.toolRun.entity.LanguageEntity +import de.fraunhofer.iem.dataprovider.toolRun.entity.ToolRunEntity +import de.fraunhofer.iem.dataprovider.toolRun.repository.LanguageRepository +import de.fraunhofer.iem.dataprovider.toolRun.repository.ToolRunRepository +import jakarta.transaction.Transactional +import org.springframework.stereotype.Service + +@Service +class ToolRunDbService( + private val toolRunRepository: ToolRunRepository, + private val languageRepository: LanguageRepository, + private val repositoryService: RepositoryService, + private val toolService: ToolService +) { + + + @Transactional + fun createToolRun( + repoDto: RepositoryCreateDto, + kpiToolList: List<Pair<CreateToolDto, List<RawValueKpiCreateDto>>>, + languageMap: Map<String, Float> + ) { + val repo = repositoryService.getOrCreate(repoDto) + val toolRun = ToolRunEntity(repository = repo) + + val languages = + languageMap.map { + LanguageEntity( + toolRunEntity = toolRun, + name = it.key, + percentage = it.value + ) + } + toolRun.languageEntities.addAll(languages) + + kpiToolList.forEach { toolKpi -> + val tool = toolService.findOrCreateTool(toolKpi.first) + toolRun.toolEntities.add(tool) + + val kpis = toolKpi.second.map { rawKpi -> + rawKpi.toDbObject(toolRun) + } + toolRun.kpiEntities.addAll(kpis) + } + + + toolRunRepository.save(toolRun) + } + + + //TODO: this should return a create tool run dto + fun getToolRunByProjectId(projectId: Long): ToolRunEntity { + return toolRunRepository.findFirstByRepository_ProjectIdOrderByCreatedAtDesc(projectId) + } +} diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt index 53cc23e6..1734266b 100644 --- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt @@ -2,15 +2,10 @@ package de.fraunhofer.iem.dataprovider.toolRun.service import de.fraunhofer.iem.dataprovider.kpi.service.KPIService import de.fraunhofer.iem.dataprovider.logger.getLogger -import de.fraunhofer.iem.dataprovider.repository.entity.RepositoryEntity import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService import de.fraunhofer.iem.dataprovider.tool.dto.ToolResponseDto import de.fraunhofer.iem.dataprovider.tool.enumeration.ToolType import de.fraunhofer.iem.dataprovider.toolRun.dto.ToolRunDto -import de.fraunhofer.iem.dataprovider.toolRun.entity.LanguageEntity -import de.fraunhofer.iem.dataprovider.toolRun.entity.ToolRunEntity -import de.fraunhofer.iem.dataprovider.toolRun.repository.LanguageRepository -import de.fraunhofer.iem.dataprovider.toolRun.repository.ToolRunRepository import de.fraunhofer.iem.dataprovider.tools.gitlab.service.RepositoryDetailsService import de.fraunhofer.iem.dataprovider.tools.occmd.service.OccmdService import de.fraunhofer.iem.dataprovider.tools.ort.service.OrtService @@ -19,13 +14,13 @@ import org.springframework.stereotype.Service @Service class ToolRunService( - private val toolRunRepository: ToolRunRepository, - private val languageRepository: LanguageRepository, - private val repositoryService: RepositoryService, + private val repositoryDetailsService: RepositoryDetailsService, private val kpiService: KPIService, private val ortService: OrtService, - private val occmdService: OccmdService + private val occmdService: OccmdService, + private val toolRunDbService: ToolRunDbService, + private val repositoryService: RepositoryService ) { private val defaultScope = CoroutineScope(Dispatchers.Default) @@ -54,13 +49,10 @@ class ToolRunService( * */ logger.info("Starting tool run creation for repo $projectId") - val toolRun = async(ioScope.coroutineContext) { - val repoDto = repositoryService.getRepositoryInfo(projectId) - val repo = repositoryService.getOrCreate(repoDto) - val tr = createToolRunForRepository(repo) - tr + val repoDto = async(ioScope.coroutineContext) { + repositoryService.getRepositoryInfoFromGitlab(projectId) }.await() - logger.info("Finished tool run creation for repo $projectId $toolRun") + logger.info("Finished tool run creation for repo $projectId $repoDto") /** * For all tools, we in parallel query the tool results API. @@ -76,24 +68,17 @@ class ToolRunService( async { val vulnerabilityDtos = ortService.getOrtResults(projectId) // in the dev setup we get results for repo id 106 - if (vulnerabilityDtos.isNotEmpty()) { - toolRun.toolEntities.add(ortService.toolEntity) - } - kpiService.calculateVulnerabilityKpis(vulnerabilityDtos) + Pair(ortService.getToolDto(), kpiService.calculateVulnerabilityKpis(vulnerabilityDtos)) }, async { val repoDetailsDto = repositoryDetailsService.getRepositoryDetails(projectId) - toolRun.toolEntities.add(repositoryDetailsService.toolEntity) - kpiService.calculateRepositoryDetailsKpis(repoDetailsDto) + Pair(repositoryDetailsService.getToolDto(), kpiService.calculateRepositoryDetailsKpis(repoDetailsDto)) }, async { - val rawOccmdResults = occmdService.runOccmd(projectId, toolRun.repository.url) - if (rawOccmdResults.isNotEmpty()) { - toolRun.toolEntities.add(occmdService.toolEntity) - } - kpiService.calculateOccmdKpis(rawOccmdResults) + val rawOccmdResults = occmdService.runOccmd(projectId, repoDto.uri) + Pair(occmdService.getToolDto(), kpiService.calculateOccmdKpis(rawOccmdResults)) } ) @@ -109,42 +94,25 @@ class ToolRunService( logger.error("API job failed with error $exception") null } - }.flatten() + } logger.info("All API queries finished for repo $projectId. Calculated ${kpis.size} raw KPIs.") - try { + val languageToPercentageMap = try { async { repositoryDetailsService.getRepositoryLanguages(projectId) }.await() } catch (exception: Exception) { emptyMap() - }.map { - languageRepository.save( - LanguageEntity( - toolRunEntity = toolRun, - name = it.key, - percentage = it.value - ) - ) - }.toMutableSet() - - if (kpis.isNotEmpty()) { - logger.info("Purging and storing new KPIs for repo $projectId") - val kpiEntities = kpiService.saveRawKpis(toolRun, kpis) - toolRun.kpiEntities.addAll(kpiEntities) - saveToolRunEntity(toolRun) } - logger.info("Finished processing repository changed request for repository $projectId") - } + toolRunDbService.createToolRun( + repoDto = repoDto, + languageMap = languageToPercentageMap, + kpiToolList = kpis + ) - fun createToolRunForRepository(repo: RepositoryEntity): ToolRunEntity { - val tr = ToolRunEntity(repository = repo) - return toolRunRepository.save(tr) + logger.info("Finished processing repository changed request for repository $projectId") } - fun saveToolRunEntity(toolRunEntity: ToolRunEntity): ToolRunEntity { - return toolRunRepository.save(toolRunEntity) - } /** * This method retrieves the tool run stored for the given repository. @@ -152,13 +120,11 @@ class ToolRunService( * in the tool run and create finding objects for them. */ suspend fun getToolRunForRepository( - repo: RepositoryEntity, + projectId: Long, dispatcher: CoroutineDispatcher = Dispatchers.IO, includeFindings: Boolean = false ): ToolRunDto { - val toolRunEntity = withContext(dispatcher) { - toolRunRepository.findFirstByRepository_IdOrderByCreatedAtDesc(repo.id!!) - } + val toolRunEntity = toolRunDbService.getToolRunByProjectId(projectId = projectId) val apiJobs: MutableList<Deferred<ToolResponseDto>> = mutableListOf() @@ -168,7 +134,7 @@ class ToolRunService( ToolType.ORT -> { apiJobs.add( defaultScope.async { - val rawOrtResult = ortService.getOrtResults(projectId = repo.projectId) + val rawOrtResult = ortService.getOrtResults(projectId = projectId) val findings = ortService.getFindings(rawOrtResult) tool.toolType.toViewModel(findings = findings) } diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/gitlab/service/RepositoryDetailsService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/gitlab/service/RepositoryDetailsService.kt index 7f189d4e..875a8720 100644 --- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/gitlab/service/RepositoryDetailsService.kt +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/gitlab/service/RepositoryDetailsService.kt @@ -3,7 +3,6 @@ package de.fraunhofer.iem.dataprovider.tools.gitlab.service import de.fraunhofer.iem.dataprovider.gitlab.service.OpenCodeGitlabApi import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryDetailsDto import de.fraunhofer.iem.dataprovider.tool.dto.CreateToolDto -import de.fraunhofer.iem.dataprovider.tool.entity.ToolEntity import de.fraunhofer.iem.dataprovider.tool.enumeration.ToolType import de.fraunhofer.iem.dataprovider.tool.service.ToolService import org.springframework.stereotype.Service @@ -19,10 +18,7 @@ class RepositoryDetailsService(private val openCodeGitlabApi: OpenCodeGitlabApi, return openCodeGitlabApi.getRepositoryLanguages(projectId) } - val toolEntity: ToolEntity = getOrCreateToolEntity() - - private final fun getOrCreateToolEntity(): ToolEntity { - val createToolDto = CreateToolDto("Gitlab API", type = ToolType.Gitlab) - return toolService.findOrCreateTool(createToolDto) + fun getToolDto(): CreateToolDto { + return CreateToolDto("Gitlab API", type = ToolType.Gitlab) } } diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt index c06068d8..13eb2ae6 100644 --- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/occmd/service/OccmdService.kt @@ -4,9 +4,7 @@ import de.fraunhofer.iem.dataprovider.configuration.DirectoryPathsProperties import de.fraunhofer.iem.dataprovider.configuration.OpenCodeGitlabApiProperties import de.fraunhofer.iem.dataprovider.logger.getLogger import de.fraunhofer.iem.dataprovider.tool.dto.CreateToolDto -import de.fraunhofer.iem.dataprovider.tool.entity.ToolEntity import de.fraunhofer.iem.dataprovider.tool.enumeration.ToolType -import de.fraunhofer.iem.dataprovider.tool.service.ToolService import de.fraunhofer.iem.dataprovider.tools.occmd.json.RawResultJson import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.Dispatchers @@ -29,16 +27,11 @@ import kotlin.io.path.deleteRecursively class OccmdService( private val dirProperties: DirectoryPathsProperties, private val gitlabApiProperties: OpenCodeGitlabApiProperties, - private val toolService: ToolService ) { val logger = getLogger(javaClass) - - val toolEntity: ToolEntity = getOrCreateToolEntity() - - private final fun getOrCreateToolEntity(): ToolEntity { - val createToolDto = CreateToolDto("OCCMD", type = ToolType.OCCMD) - return toolService.findOrCreateTool(createToolDto) + fun getToolDto(): CreateToolDto { + return CreateToolDto("OCCMD", type = ToolType.OCCMD) } /** diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/ort/service/OrtService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/ort/service/OrtService.kt index e86132e6..08ffe3c9 100644 --- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/ort/service/OrtService.kt +++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/tools/ort/service/OrtService.kt @@ -4,9 +4,7 @@ import de.fraunhofer.iem.dataprovider.configuration.OpenCodeApiProperties import de.fraunhofer.iem.dataprovider.logger.getLogger import de.fraunhofer.iem.dataprovider.tool.dto.CreateToolDto import de.fraunhofer.iem.dataprovider.tool.dto.FindingDto -import de.fraunhofer.iem.dataprovider.tool.entity.ToolEntity import de.fraunhofer.iem.dataprovider.tool.enumeration.ToolType -import de.fraunhofer.iem.dataprovider.tool.service.ToolService import de.fraunhofer.iem.dataprovider.tools.ort.dto.VulnerabilityDto import de.fraunhofer.iem.dataprovider.tools.ort.json.OrtJson import de.fraunhofer.iem.dataprovider.tools.ort.json.ResultJson @@ -20,14 +18,12 @@ import org.springframework.stereotype.Service @Service class OrtService( private val openCodeApiProperties: OpenCodeApiProperties, - private val toolService: ToolService, httpClientWrapper: HttpClientWrapper ) { private val logger = getLogger(javaClass) private val httpClient = httpClientWrapper.getClient() - val toolEntity: ToolEntity = getOrCreateToolEntity() /** * Queries the ORT API and transforms the received JSON objects into @@ -57,9 +53,8 @@ class OrtService( } } - private final fun getOrCreateToolEntity(): ToolEntity { - val createToolDto = CreateToolDto("ORT", ToolType.ORT) - return toolService.findOrCreateTool(createToolDto) + fun getToolDto(): CreateToolDto { + return CreateToolDto("ORT", ToolType.ORT) } private suspend fun queryOrtApi(projectId: Long): List<ResultJson> { -- GitLab