From 9dd743c970d7fd8ef24d8204d9ff72d83f893e57 Mon Sep 17 00:00:00 2001
From: Jan-Niclas Struewer <j.n.struewer@gmail.com>
Date: Fri, 22 Sep 2023 18:06:59 +0200
Subject: [PATCH] started to implement new logic to coordinate the api queries
and KPI calculation
---
.../gitlab/controller/GitlabController.kt | 7 +-
.../gitlab/service/OpenCodeGitlabApi.kt | 83 ++++++++
.../iem/dataprovider/kpi/dto/KPICreateDto.kt | 2 +-
.../kpi/metrics/MetricsService.kt | 72 +++++++
.../dto/RepositoryDetailsKpiDto.kt | 12 ++
.../dto/RepositoryDetailsKpisEntity.kt | 12 ++
.../service/RepositoryDetailsService.kt | 106 ++++++++++
.../dataprovider/kpi/service/KPIService.kt | 18 ++
.../repository/dto/RepositoryDetailsDto.kt | 4 +-
.../repository/service/RepositoryService.kt | 8 +-
.../dataprovider/taskManager/TaskManager.kt | 188 +++++++++---------
.../dataQuery/GetGitlabProjectInfoTask.kt | 36 ----
.../dataQuery/GetRepositoryDetailsTask.kt | 74 -------
.../tasks/kpiCalculation/MetricsTask.kt | 49 +----
.../toolRun/service/ToolRunService.kt | 4 +-
15 files changed, 411 insertions(+), 264 deletions(-)
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/service/OpenCodeGitlabApi.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/dto/RepositoryDetailsKpiDto.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/dto/RepositoryDetailsKpisEntity.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/service/RepositoryDetailsService.kt
delete mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/dataQuery/GetGitlabProjectInfoTask.kt
delete mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/dataQuery/GetRepositoryDetailsTask.kt
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/controller/GitlabController.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/controller/GitlabController.kt
index 95baf8ed..47b4fb80 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/controller/GitlabController.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/controller/GitlabController.kt
@@ -2,16 +2,15 @@ package de.fraunhofer.iem.dataprovider.gitlab.controller
import de.fraunhofer.iem.dataprovider.configuration.ApiPaths
import de.fraunhofer.iem.dataprovider.gitlab.dto.RepositoryChangedDto
+import de.fraunhofer.iem.dataprovider.kpi.metrics.MetricsService
import de.fraunhofer.iem.dataprovider.logger.getLogger
-import de.fraunhofer.iem.dataprovider.taskManager.TaskManager
-import de.fraunhofer.iem.dataprovider.taskManager.events.RepoChangedEvent
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RestController
@RestController
-class GitlabController(private val taskManager: TaskManager) {
+class GitlabController(private val metricsService: MetricsService) {
private val logger = getLogger(javaClass)
@@ -20,6 +19,6 @@ class GitlabController(private val taskManager: TaskManager) {
@PostMapping(ApiPaths.OPENCODE_REPO_CHANGED)
suspend fun repoChanged(@RequestBody repositoryChangedDto: RepositoryChangedDto) {
logger.info("Repo changed POST request for ID ${repositoryChangedDto.projectId} received.")
- taskManager.addEvent(RepoChangedEvent(repoId = repositoryChangedDto.projectId))
+ metricsService.handleRepositoryChange(repositoryChangedDto.projectId)
}
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/service/OpenCodeGitlabApi.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/service/OpenCodeGitlabApi.kt
new file mode 100644
index 00000000..161049ab
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/service/OpenCodeGitlabApi.kt
@@ -0,0 +1,83 @@
+package de.fraunhofer.iem.dataprovider.gitlab.service
+
+import de.fraunhofer.iem.dataprovider.configuration.OpenCodeGitlabApiProperties
+import de.fraunhofer.iem.dataprovider.logger.getLogger
+import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryCreateDto
+import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryDetailsDto
+import org.gitlab4j.api.GitLabApi
+import org.gitlab4j.api.models.Commit
+import org.gitlab4j.api.models.Project
+import org.springframework.stereotype.Service
+
+
+/**
+ * This class wraps all usages of org.gitlab4j and transforms their data models
+ * into our internal data models.
+ */
+@Service
+class OpenCodeGitlabApi(openCodeGitlabConfiguration: OpenCodeGitlabApiProperties) {
+ private val logger = getLogger(javaClass)
+ private val gitlabApi: GitLabApi =
+ GitLabApi(openCodeGitlabConfiguration.host, openCodeGitlabConfiguration.accessToken)
+
+ /**
+ * Queries the gitlab project API at ${gitlabConfiguration.host} and returns a RepositoryCreateDto
+ */
+ fun getRepositoryInfo(repoId: Long): RepositoryCreateDto {
+ logger.info("Get repository info for repository id $repoId started")
+
+ val project = gitlabApi.projectApi.getProject(repoId)
+ val projectUri = project.httpUrlToRepo
+ val repoCreateDto = RepositoryCreateDto(project.path, projectUri, repoId)
+
+ logger.info("Get repository info $repoCreateDto finished successfully.")
+ return repoCreateDto
+ }
+
+ /**
+ * Queries gitlab project, commit, and repository API to create a RepositoryDetailsDto
+ */
+ fun getRepositoryDetails(repoId: Long): RepositoryDetailsDto {
+ logger.info("Get repository details for repository id $repoId started")
+
+ val project = gitlabApi.projectApi.getProject(repoId)
+ // Note: We only take commits from the default branch
+ val commits = gitlabApi.commitsApi.getCommits(project.id, project.defaultBranch, ".")
+
+ val numberOfCommits = commits.count()
+ val numberOfSignedCommits = getNumberOfSignedCommits(repoId, commits)
+ val isDefaultBranchProtected = isDefaultBranchProtected(repoId, project)
+ val repoDetailsDto = RepositoryDetailsDto(
+ repoId,
+ numberOfCommits,
+ numberOfSignedCommits,
+ isDefaultBranchProtected
+ )
+
+ logger.info("Get repository details $repoDetailsDto finished successfully.")
+ return repoDetailsDto
+ }
+
+ private fun getNumberOfSignedCommits(repoId: Long, commits: List<Commit>): Int {
+ var numberOfSignedCommits = 0
+ for (commit in commits) {
+ val signature = gitlabApi.commitsApi.getOptionalGpgSignature(repoId, commit.id)
+ if (signature !== null) {
+ numberOfSignedCommits++
+ }
+ }
+ return numberOfSignedCommits
+ }
+
+ // TODO: This should probably live somewhere else and encapsulate the logic
+ private fun isDefaultBranchProtected(repoId: Long, project: Project): Boolean {
+ return try {
+ val defaultBranchName = project.defaultBranch
+ val branch = gitlabApi.repositoryApi.getBranch(repoId, defaultBranchName)
+ branch.protected
+ } catch (e: Exception) {
+ // in theory, error probably happens if branch can't be found. In this case we default to false
+ false
+ }
+ }
+}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/dto/KPICreateDto.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/dto/KPICreateDto.kt
index 0ff76fb0..2232cbd1 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/dto/KPICreateDto.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/dto/KPICreateDto.kt
@@ -13,7 +13,7 @@ class KPICreateDto(
private val calculationStrategy: KPICalculationStrategy,
private val displayScore: String? = null
) {
- val hierarchyEdges: MutableList<KPIHierarchyEdgeDto> = mutableListOf<KPIHierarchyEdgeDto>()
+ val hierarchyEdges: MutableList<KPIHierarchyEdgeDto> = mutableListOf()
var value: Int? = null
fun calculateKPI() {
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt
new file mode 100644
index 00000000..910107d2
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/MetricsService.kt
@@ -0,0 +1,72 @@
+package de.fraunhofer.iem.dataprovider.kpi.metrics
+
+import de.fraunhofer.iem.dataprovider.kpi.metrics.repositoryDetails.service.RepositoryDetailsService
+import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService
+import de.fraunhofer.iem.dataprovider.toolRun.service.ToolRunService
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.launch
+import org.springframework.stereotype.Service
+
+@Service
+class MetricsService(
+ private val repositoryService: RepositoryService,
+ private val toolRunService: ToolRunService,
+ private val repositoryDetailsService: RepositoryDetailsService
+) {
+
+ private val defaultScope = CoroutineScope(Dispatchers.Default)
+ val ioScope = CoroutineScope(Dispatchers.IO)
+
+ suspend fun handleRepositoryChange(repoId: Long) = defaultScope.launch {
+
+ /**
+ * Create initial db models.
+ * If this fails we want the whole method to fail as we can't continue
+ * without the db models.
+ *
+ */
+ val toolRun = async(ioScope.coroutineContext) {
+ val repoDto = repositoryService.getRepositoryInfo(repoId)
+ val repo = repositoryService.getOrCreate(repoDto)
+ toolRunService.createToolRunForRepository(repo)
+ }.await()
+
+ /**
+ * For all tools, first query the tool results API.
+ * If the tool has results, add the tool to the tool run
+ * and then calculate the KPI based on the results, link
+ * the KPI with the repository and store everything in the db.
+ * The function returns the UUID of the KPI in the db.
+ * If we can calculate multiple KPIs from one tool we can return
+ * a list of UUIDs.
+ *
+ * val toolKpi = async {
+ * val toolResult = queryToolApi(repoId)
+ * if (!toolResult.isEmpty) {
+ * toolRun.addTool(tool)
+ * val kpi = calculateKpi(toolResults)
+ * kpi.addRepository(toolRun.repo)
+ * kpi.store()
+ * kpi.id // this is the return of the async function
+ * }
+ * }
+ */
+
+
+ val repositoryDetailsKpi = async {
+ val repoDetailsDto = repositoryDetailsService.getRepositoryDetails(repoId)
+ val repositoryDetailsKpiDto = repositoryDetailsService.calculateRepositoryDetailsKpis(repoDetailsDto)
+
+ repositoryDetailsService.saveRepositoryDetailsKpi(toolRun.repository!!, repositoryDetailsKpiDto)
+ }
+
+ /**
+ * Calculate KPI tree by awaiting all tool KPIs
+ * joinAll(toolKpi)
+ * calculateKpiTree(toolKpi, ...)
+ */
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/dto/RepositoryDetailsKpiDto.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/dto/RepositoryDetailsKpiDto.kt
new file mode 100644
index 00000000..59717339
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/dto/RepositoryDetailsKpiDto.kt
@@ -0,0 +1,12 @@
+package de.fraunhofer.iem.dataprovider.kpi.metrics.repositoryDetails.dto
+
+import de.fraunhofer.iem.dataprovider.kpi.dto.KPICreateDto
+
+data class RepositoryDetailsKpisDto(
+ val numberOfCommitsKpi: KPICreateDto,
+ val numberOfSignedCommitsKPI: KPICreateDto,
+ val isDefaultBranchProtectedKPI: KPICreateDto,
+ val signedCommitsRatioKPI: KPICreateDto,
+ val processComplianceKPI: KPICreateDto,
+ val processTransparencyKPI: KPICreateDto
+)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/dto/RepositoryDetailsKpisEntity.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/dto/RepositoryDetailsKpisEntity.kt
new file mode 100644
index 00000000..c792f480
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/dto/RepositoryDetailsKpisEntity.kt
@@ -0,0 +1,12 @@
+package de.fraunhofer.iem.dataprovider.kpi.metrics.repositoryDetails.dto
+
+import de.fraunhofer.iem.dataprovider.kpi.entity.KPIEntity
+
+data class RepositoryDetailsKpisEntity(
+ val numberOfCommitsKpi: KPIEntity,
+ val numberOfSignedCommitsKPI: KPIEntity,
+ val isDefaultBranchProtectedKPI: KPIEntity,
+ val signedCommitsRatioKPI: KPIEntity,
+ val processComplianceKPI: KPIEntity,
+ val processTransparencyKPI: KPIEntity
+)
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/service/RepositoryDetailsService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/service/RepositoryDetailsService.kt
new file mode 100644
index 00000000..9021c961
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/metrics/repositoryDetails/service/RepositoryDetailsService.kt
@@ -0,0 +1,106 @@
+package de.fraunhofer.iem.dataprovider.kpi.metrics.repositoryDetails.service
+
+import de.fraunhofer.iem.dataprovider.gitlab.service.OpenCodeGitlabApi
+import de.fraunhofer.iem.dataprovider.kpi.dto.KPICreateDto
+import de.fraunhofer.iem.dataprovider.kpi.metrics.repositoryDetails.dto.RepositoryDetailsKpisDto
+import de.fraunhofer.iem.dataprovider.kpi.metrics.repositoryDetails.dto.RepositoryDetailsKpisEntity
+import de.fraunhofer.iem.dataprovider.kpi.service.KPIService
+import de.fraunhofer.iem.dataprovider.kpi.strategy.AggregationKPICalculationStrategy
+import de.fraunhofer.iem.dataprovider.kpi.strategy.RatioKPICalculationStrategy
+import de.fraunhofer.iem.dataprovider.kpi.strategy.RawValueKPICalculationStrategy
+import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryDetailsDto
+import de.fraunhofer.iem.dataprovider.repository.entity.RepositoryEntity
+import org.springframework.stereotype.Service
+
+@Service
+class RepositoryDetailsService(private val kpiService: KPIService, private val openCodeGitlabApi: OpenCodeGitlabApi) {
+
+ fun getRepositoryDetails(repoId: Long): RepositoryDetailsDto {
+ return openCodeGitlabApi.getRepositoryDetails(repoId)
+ }
+
+ fun calculateRepositoryDetailsKpis(repoDetailsDto: RepositoryDetailsDto): RepositoryDetailsKpisDto {
+ val numberOfCommitsKPI = KPICreateDto(
+ "Number of Commits",
+ "Total number of commits on the default branch of the repository.",
+ false,
+ RawValueKPICalculationStrategy(repoDetailsDto.numberOfCommits)
+ )
+ val numberOfSignedCommitsKPI = KPICreateDto(
+ "Number of Signed Commits",
+ "Total number of signed and verified commits on the default branch of the repository.",
+ false,
+ RawValueKPICalculationStrategy(repoDetailsDto.numberOfSignedCommits)
+ )
+ val isDefaultBranchProtectedKPI = KPICreateDto(
+ "Default Branch Protection",
+ "Used to assess compliance with a standard development process. For this purpose, it is examined whether the standard development branch is protected against unintentional changes.",
+ false,
+ RawValueKPICalculationStrategy(if (repoDetailsDto.isDefaultBranchProtected) 100 else 0)
+ )
+
+ val signedCommitsRatioKPI =
+ KPICreateDto(
+ "Commit Signature Ratio",
+ "Used to assess compliance with a common and transparent development process. It is desirable that all commits are signed by their authors. Therefore, the ratio of signed commits to all commits is determined to calculate this metric.",
+ false,
+ RatioKPICalculationStrategy()
+ )
+ signedCommitsRatioKPI.addChildKPI(numberOfCommitsKPI, 1.0)
+ signedCommitsRatioKPI.addChildKPI(numberOfSignedCommitsKPI, 1.0)
+
+ val processComplianceKPI = KPICreateDto(
+ "Process Compliance Score",
+ "Assesses the development process of the software provided. For this purpose, the development process traceable in the repository is compared with common development standards to enable an assessment.",
+ false,
+ AggregationKPICalculationStrategy()
+ )
+
+ val processTransparencyKPI = KPICreateDto(
+ "Process Transparency Score",
+ "Assesses the transparency resp. traceability of the development process of the provided software for external parties. For this purpose, various analyzes are performed that assess the availability of information about the software development process within the repository.",
+ false,
+ AggregationKPICalculationStrategy()
+ )
+ processComplianceKPI.addChildKPI(isDefaultBranchProtectedKPI, 0.5)
+ processComplianceKPI.addChildKPI(signedCommitsRatioKPI, 0.5)
+ processTransparencyKPI.addChildKPI(signedCommitsRatioKPI, 1.0)
+
+ return RepositoryDetailsKpisDto(
+ processComplianceKPI = processComplianceKPI,
+ processTransparencyKPI = processTransparencyKPI,
+ signedCommitsRatioKPI = signedCommitsRatioKPI,
+ isDefaultBranchProtectedKPI = isDefaultBranchProtectedKPI,
+ numberOfSignedCommitsKPI = numberOfSignedCommitsKPI,
+ numberOfCommitsKpi = numberOfCommitsKPI
+ )
+ }
+
+
+ fun saveRepositoryDetailsKpi(
+ repositoryEntity: RepositoryEntity,
+ repositoryDetailsKpisDto: RepositoryDetailsKpisDto
+ ): RepositoryDetailsKpisEntity {
+ return RepositoryDetailsKpisEntity(
+ numberOfCommitsKpi = kpiService.saveKpi(repositoryEntity, repositoryDetailsKpisDto.numberOfCommitsKpi),
+ numberOfSignedCommitsKPI = kpiService.saveKpi(
+ repositoryEntity,
+ repositoryDetailsKpisDto.numberOfSignedCommitsKPI
+ ),
+ isDefaultBranchProtectedKPI = kpiService.saveKpi(
+ repositoryEntity,
+ repositoryDetailsKpisDto.isDefaultBranchProtectedKPI
+ ),
+ signedCommitsRatioKPI = kpiService.saveKpi(
+ repositoryEntity,
+ repositoryDetailsKpisDto.signedCommitsRatioKPI
+ ),
+ processComplianceKPI = kpiService.saveKpi(repositoryEntity, repositoryDetailsKpisDto.processComplianceKPI),
+ processTransparencyKPI = kpiService.saveKpi(
+ repositoryEntity,
+ repositoryDetailsKpisDto.processTransparencyKPI
+ ),
+
+ )
+ }
+}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt
index 2d46275a..468d25b5 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/kpi/service/KPIService.kt
@@ -19,6 +19,24 @@ class KPIService(
private val logger = getLogger(javaClass)
+
+ /**
+ * Stores the given KPIs and links them to the given repository
+ */
+ fun saveKpis(repositoryEntity: RepositoryEntity, createKpis: Array<KPICreateDto>): Array<KPIEntity> {
+ return createKpis.map { saveKpi(repositoryEntity, it) }.toTypedArray()
+ }
+
+
+ /**
+ * Stores the given KPI and links it to the given repository
+ */
+ fun saveKpi(repositoryEntity: RepositoryEntity, createKpi: KPICreateDto): KPIEntity {
+ val kpiEntity = createKpi.toDbObject()
+ kpiEntity.repository = repositoryEntity
+ return kpiRepository.save(kpiEntity)
+ }
+
private fun calculateKPIsRecursively(kpi: KPICreateDto, visited: MutableSet<KPICreateDto>) {
// Check if the KPI has already been visited
if (visited.contains(kpi)) {
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/dto/RepositoryDetailsDto.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/dto/RepositoryDetailsDto.kt
index 317aef4e..0b876c99 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/dto/RepositoryDetailsDto.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/dto/RepositoryDetailsDto.kt
@@ -1,9 +1,7 @@
package de.fraunhofer.iem.dataprovider.repository.dto
-import de.fraunhofer.iem.dataprovider.repository.entity.RepositoryEntity
-
data class RepositoryDetailsDto(
- val repositoryEntity: RepositoryEntity,
+ val repoId: Long,
val numberOfCommits: Int,
val numberOfSignedCommits: Int,
val isDefaultBranchProtected: Boolean
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/service/RepositoryService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/service/RepositoryService.kt
index cb38f1b3..da31c5f0 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/service/RepositoryService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/repository/service/RepositoryService.kt
@@ -1,5 +1,6 @@
package de.fraunhofer.iem.dataprovider.repository.service
+import de.fraunhofer.iem.dataprovider.gitlab.service.OpenCodeGitlabApi
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryCreateDto
import de.fraunhofer.iem.dataprovider.repository.entity.RepositoryEntity
@@ -10,10 +11,11 @@ import java.util.*
@Service
class RepositoryService(
private val repositoryRepository: RepositoryRepository,
+ private val openCodeGitlabApi: OpenCodeGitlabApi
) {
private val logger = getLogger(javaClass)
- fun getRepoOrCreate(gitRepository: RepositoryCreateDto): RepositoryEntity {
+ fun getOrCreate(gitRepository: RepositoryCreateDto): RepositoryEntity {
var repo = repositoryRepository.findByRepoId(gitRepository.id)
if (repo == null) {
repo = repositoryRepository.save(gitRepository.toDbObject())
@@ -21,6 +23,10 @@ class RepositoryService(
return repo
}
+ fun getRepositoryInfo(repoId: Long): RepositoryCreateDto {
+ return openCodeGitlabApi.getRepositoryInfo(repoId)
+ }
+
// TODO: We should decide if we want to return optionals (null if not found) or throw exceptions or ...
// TODO: Must be consistent across all methods
fun findRepoByID(id: UUID): RepositoryEntity? {
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
index 7aff78c2..d4241779 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
@@ -5,11 +5,7 @@ import de.fraunhofer.iem.dataprovider.configuration.OpenCodeGitlabApiProperties
import de.fraunhofer.iem.dataprovider.kpi.service.KPIService
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService
-import de.fraunhofer.iem.dataprovider.taskManager.events.*
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.dataQuery.GetGitlabProjectTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.dataQuery.GetRepositoryDetailsTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.kpiCalculation.MetricsTask
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.tools.ort.OrtApiTask
+import de.fraunhofer.iem.dataprovider.taskManager.events.Event
import de.fraunhofer.iem.dataprovider.tool.service.ToolService
import de.fraunhofer.iem.dataprovider.toolRun.service.ToolRunService
import jakarta.annotation.PostConstruct
@@ -73,104 +69,104 @@ class TaskManager(
private val groupTaskManager = GroupTaskManager(::onReceive)
suspend fun onReceive(event: Event) {
logger.info("[${Thread.currentThread().name}] add task called in Task Manager $event")
-
- when (event) {
- is RepoChangedEvent -> {
- ioWorker.addTask(
- GetGitlabProjectTask(
- event.repoId,
- openCodeGitlabConfiguration,
- ::addEvent,
- repositoryService,
- )
- )
- }
-
- is GetGitlabProjectDoneEvent -> {
- // TODO: we should replace the git clone task with the temporary OCCMD task, as only OCCMD
-
- val groupId = groupTaskManager.createTaskGroup(event.repoId)
-
-// val occmdTask = OccmdTask(
-// event.outputDirectory,
-// config.toolResultsTargetDirectory,
+//
+// when (event) {
+// is RepoChangedEvent -> {
+// ioWorker.addTask(
+// GetGitlabProjectTask(
+// event.repoId,
// openCodeGitlabConfiguration,
+// ::addEvent,
// repositoryService,
+// )
+// )
+// }
+//
+// is GetGitlabProjectDoneEvent -> {
+// // TODO: we should replace the git clone task with the temporary OCCMD task, as only OCCMD
+//
+// val groupId = groupTaskManager.createTaskGroup(event.repoId)
+//
+//// val occmdTask = OccmdTask(
+//// event.outputDirectory,
+//// config.toolResultsTargetDirectory,
+//// openCodeGitlabConfiguration,
+//// repositoryService,
+//// ::addEvent,
+//// event.repoId,
+//// groupId,
+//// toolRunService
+//// )
+// ioWorker.addTask(
+// OrtApiTask(
// ::addEvent,
// event.repoId,
-// groupId,
+// toolService,
+// repositoryService
+// )
+// )
+//
+//// groupTaskManager.addTaskToGroup(groupId, occmdTask.taskID)
+////
+//// worker.addTask(occmdTask)
+// }
+//
+// is GroupTaskDoneEvent -> {
+// logger.info("GroupTaskDoneEvent received")
+// groupTaskManager.taskInGroupFinished(event.groupId, event.taskId)
+// }
+//
+// is GroupTasksDoneEvent -> {
+// logger.info("Adding repository details task")
+// worker.addTask(
+// GetRepositoryDetailsTask(
+// event.repoId,
+// openCodeGitlabConfiguration,
+// repositoryService,
+// ::addEvent
+// )
+// )
+// }
+//
+// is GetRepositoryDetailsDoneEvent -> {
+// worker.addTask(
+// MetricsTask(
+// event.repoId,
+// repositoryService,
+// kpiService,
+// ::addEvent,
// toolRunService
// )
- ioWorker.addTask(
- OrtApiTask(
- ::addEvent,
- event.repoId,
- toolService,
- repositoryService
- )
- )
-
-// groupTaskManager.addTaskToGroup(groupId, occmdTask.taskID)
+// )
+// }
//
-// worker.addTask(occmdTask)
- }
-
- is GroupTaskDoneEvent -> {
- logger.info("GroupTaskDoneEvent received")
- groupTaskManager.taskInGroupFinished(event.groupId, event.taskId)
- }
-
- is GroupTasksDoneEvent -> {
- logger.info("Adding repository details task")
- worker.addTask(
- GetRepositoryDetailsTask(
- event.repoId,
- openCodeGitlabConfiguration,
- repositoryService,
- ::addEvent
- )
- )
- }
-
- is GetRepositoryDetailsDoneEvent -> {
- worker.addTask(
- MetricsTask(
- event.repoId,
- repositoryService,
- kpiService,
- ::addEvent,
- toolRunService
- )
- )
- }
-
- is RecalculateAllKpisEvent -> {
- repositoryService.getAllRepositories().forEach { repo ->
- worker.addTask(
- MetricsTask(
- repo.id!!,
- repositoryService,
- kpiService,
- ::addEvent,
- toolRunService
- )
- )
- }
- }
-
- is TaskFailedEvent -> {
- logger.info("task failed event received. ${event.throwable}")
- }
-
- is GroupTaskFailedEvent -> {
- logger.info("Group task failed event received. ${event.throwable} ")
- groupTaskManager.taskInGroupFinished(event.groupId, event.taskId)
- }
-
- else -> {
- logger.info("Received event without special handling associated $event")
- }
- }
+// is RecalculateAllKpisEvent -> {
+// repositoryService.getAllRepositories().forEach { repo ->
+// worker.addTask(
+// MetricsTask(
+// repo.id!!,
+// repositoryService,
+// kpiService,
+// ::addEvent,
+// toolRunService
+// )
+// )
+// }
+// }
+//
+// is TaskFailedEvent -> {
+// logger.info("task failed event received. ${event.throwable}")
+// }
+//
+// is GroupTaskFailedEvent -> {
+// logger.info("Group task failed event received. ${event.throwable} ")
+// groupTaskManager.taskInGroupFinished(event.groupId, event.taskId)
+// }
+//
+// else -> {
+// logger.info("Received event without special handling associated $event")
+// }
+// }
logger.info("[${Thread.currentThread().name}] add task finished in Task Manager $event")
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/dataQuery/GetGitlabProjectInfoTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/dataQuery/GetGitlabProjectInfoTask.kt
deleted file mode 100644
index de89c47a..00000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/dataQuery/GetGitlabProjectInfoTask.kt
+++ /dev/null
@@ -1,36 +0,0 @@
-package de.fraunhofer.iem.dataprovider.taskManager.tasks.dataQuery
-
-import de.fraunhofer.iem.dataprovider.configuration.OpenCodeGitlabApiProperties
-import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryCreateDto
-import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService
-import de.fraunhofer.iem.dataprovider.taskManager.events.Event
-import de.fraunhofer.iem.dataprovider.taskManager.events.GetGitlabProjectDoneEvent
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.Task
-import org.gitlab4j.api.GitLabApi
-
-
-class GetGitlabProjectTask(
- private val repoId: Long,
- gitlabConfiguration: OpenCodeGitlabApiProperties,
- override val responseChannel: suspend (event: Event) -> Unit,
- private val repositoryService: RepositoryService
-) : Task() {
-
-
- private val gitlabApi: GitLabApi = GitLabApi(gitlabConfiguration.host, gitlabConfiguration.accessToken)
-
- override suspend fun execute() {
-
- val project = gitlabApi.projectApi.getProject(repoId)
- logger.info(project.toString())
- val projectUri = project.httpUrlToRepo
-
- val gitRepository = RepositoryCreateDto(project.path, projectUri, repoId)
-
- val repo = repositoryService.getRepoOrCreate(gitRepository)
-
- logger.info("Retrieved project ${project.path} and url $projectUri")
-
- responseChannel(GetGitlabProjectDoneEvent(taskID, repo.id!!, gitRepository))
- }
-}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/dataQuery/GetRepositoryDetailsTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/dataQuery/GetRepositoryDetailsTask.kt
deleted file mode 100644
index 46c01434..00000000
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/dataQuery/GetRepositoryDetailsTask.kt
+++ /dev/null
@@ -1,74 +0,0 @@
-package de.fraunhofer.iem.dataprovider.taskManager.tasks.dataQuery
-
-import de.fraunhofer.iem.dataprovider.configuration.OpenCodeGitlabApiProperties
-import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryDetailsDto
-import de.fraunhofer.iem.dataprovider.repository.entity.RepositoryEntity
-import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService
-import de.fraunhofer.iem.dataprovider.taskManager.events.Event
-import de.fraunhofer.iem.dataprovider.taskManager.events.GetRepositoryDetailsDoneEvent
-import de.fraunhofer.iem.dataprovider.taskManager.tasks.Task
-import org.gitlab4j.api.GitLabApi
-import org.gitlab4j.api.models.Commit
-import org.gitlab4j.api.models.Project
-import java.util.*
-
-class GetRepositoryDetailsTask(
- private val repoId: UUID,
- gitlabConfiguration: OpenCodeGitlabApiProperties,
- private val repositoryService: RepositoryService,
- override val responseChannel: suspend (event: Event) -> Unit,
-) : Task() {
- private val gitlabApi: GitLabApi = GitLabApi(gitlabConfiguration.host, gitlabConfiguration.accessToken)
-
- override suspend fun execute() {
- val repositoryEntity = repositoryService.findRepoByID(repoId)
- logger.info(repositoryEntity.toString())
-
- if (repositoryEntity != null) {
- val project = gitlabApi.projectApi.getProject(repositoryEntity.repoId)
- // Note: We only take commits from the default branch
- val commits = gitlabApi.commitsApi.getCommits(project.id, project.defaultBranch, ".")
-
- val numberOfCommits = commits.count()
- val numberOfSignedCommits = getNumberOfSignedCommits(repositoryEntity, commits)
- val isDefaultBranchProtected = isDefaultBranchProtected(repositoryEntity, project)
- val repositoryDetailsDto = RepositoryDetailsDto(
- repositoryEntity,
- numberOfCommits,
- numberOfSignedCommits,
- isDefaultBranchProtected
- )
-
- logger.info("Collected repository details from $repoId (${repositoryDetailsDto}) successfully")
- //TODO: WE DON'T WANT TO STORE THE RESULT IN THE DB. WE NOW DIRECTLY WANT TO CALCULATE THE KPI!
-
- responseChannel(GetRepositoryDetailsDoneEvent(taskID, repoId, repositoryDetailsDto))
- } else {
- logger.error("Repository $repoId can not be found in the database")
- }
- }
-
- // TODO: This should probably live somewhere else and encapsulate the logic
- private fun getNumberOfSignedCommits(repositoryEntity: RepositoryEntity, commits: List<Commit>): Int {
- var numberOfSignedCommits = 0
- for (commit in commits) {
- val signature = gitlabApi.commitsApi.getOptionalGpgSignature(repositoryEntity.repoId, commit.id)
- if (signature !== null) {
- numberOfSignedCommits++
- }
- }
- return numberOfSignedCommits
- }
-
- // TODO: This should probably live somewhere else and encapsulate the logic
- private fun isDefaultBranchProtected(repositoryEntity: RepositoryEntity, project: Project): Boolean {
- return try {
- val defaultBranchName = project.defaultBranch
- val branch = gitlabApi.repositoryApi.getBranch(repositoryEntity.repoId, defaultBranchName)
- branch.protected
- } catch (e: Exception) {
- // in theory, error probably happens if branch can't be found. In this case we default to false
- false
- }
- }
-}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/kpiCalculation/MetricsTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/kpiCalculation/MetricsTask.kt
index ca487224..59d5df59 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/kpiCalculation/MetricsTask.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/kpiCalculation/MetricsTask.kt
@@ -4,7 +4,6 @@ import de.fraunhofer.iem.dataprovider.kpi.dto.KPICreateDto
import de.fraunhofer.iem.dataprovider.kpi.service.KPIService
import de.fraunhofer.iem.dataprovider.kpi.strategy.AggregationKPICalculationStrategy
import de.fraunhofer.iem.dataprovider.kpi.strategy.MaximumKPICalculationStrategy
-import de.fraunhofer.iem.dataprovider.kpi.strategy.RatioKPICalculationStrategy
import de.fraunhofer.iem.dataprovider.kpi.strategy.RawValueKPICalculationStrategy
import de.fraunhofer.iem.dataprovider.repository.dto.RepositoryDetailsDto
import de.fraunhofer.iem.dataprovider.repository.service.RepositoryService
@@ -42,35 +41,7 @@ class MetricsTask(
}
private fun KPITreeFactory(repositoryDetailsEntity: RepositoryDetailsDto): KPICreateDto {
- // lowest level leaves
- val numberOfCommitsKPI = KPICreateDto(
- "Number of Commits",
- "Total number of commits on the default branch of the repository.",
- false,
- RawValueKPICalculationStrategy(repositoryDetailsEntity.numberOfCommits!!)
- )
- val numberOfSignedCommitsKPI = KPICreateDto(
- "Number of Signed Commits",
- "Total number of signed and verified commits on the default branch of the repository.",
- false,
- RawValueKPICalculationStrategy(repositoryDetailsEntity.numberOfSignedCommits!!)
- )
- val isDefaultBranchProtectedKPI = KPICreateDto(
- "Default Branch Protection",
- "Used to assess compliance with a standard development process. For this purpose, it is examined whether the standard development branch is protected against unintentional changes.",
- false,
- RawValueKPICalculationStrategy(if (repositoryDetailsEntity.isDefaultBranchProtected == true) 100 else 0)
- )
- val signedCommitsRatioKPI =
- KPICreateDto(
- "Commit Signature Ratio",
- "Used to assess compliance with a common and transparent development process. It is desirable that all commits are signed by their authors. Therefore, the ratio of signed commits to all commits is determined to calculate this metric.",
- false,
- RatioKPICalculationStrategy()
- )
- signedCommitsRatioKPI.addChildKPI(numberOfCommitsKPI, 1.0)
- signedCommitsRatioKPI.addChildKPI(numberOfSignedCommitsKPI, 1.0)
// second level
// TODO: @Jan-Niclas Please add the dependency kpis to the security KPI
@@ -108,22 +79,6 @@ class MetricsTask(
// securityKPI.addChildKPI(secretKpi, 0.5)
- val processComplianceKPI = KPICreateDto(
- "Process Compliance Score",
- "Assesses the development process of the software provided. For this purpose, the development process traceable in the repository is compared with common development standards to enable an assessment.",
- false,
- AggregationKPICalculationStrategy()
- )
-
- val processTransparencyKPI = KPICreateDto(
- "Process Transparency Score",
- "Assesses the transparency resp. traceability of the development process of the provided software for external parties. For this purpose, various analyzes are performed that assess the availability of information about the software development process within the repository.",
- false,
- AggregationKPICalculationStrategy()
- )
- processComplianceKPI.addChildKPI(isDefaultBranchProtectedKPI, 0.5)
- processComplianceKPI.addChildKPI(signedCommitsRatioKPI, 0.5)
- processTransparencyKPI.addChildKPI(signedCommitsRatioKPI, 1.0)
val rootKPI = KPICreateDto(
"Project Score",
@@ -131,8 +86,8 @@ class MetricsTask(
true,
AggregationKPICalculationStrategy()
)
- rootKPI.addChildKPI(processTransparencyKPI, 0.25)
- rootKPI.addChildKPI(processComplianceKPI, 0.25)
+// rootKPI.addChildKPI(processTransparencyKPI, 0.25)
+// rootKPI.addChildKPI(processComplianceKPI, 0.25)
rootKPI.addChildKPI(securityKPI, 0.5)
return rootKPI
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt
index 0f496576..f9c5b430 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/service/ToolRunService.kt
@@ -11,10 +11,10 @@ class ToolRunService(private val toolRunRepository: ToolRunRepository) {
private val logger = getLogger(javaClass)
- fun createToolRunForRepository(repo: RepositoryEntity) {
+ fun createToolRunForRepository(repo: RepositoryEntity): ToolRunEntity {
val tr = ToolRunEntity()
tr.repository = repo
- toolRunRepository.save(tr)
+ return toolRunRepository.save(tr)
}
fun getLatestToolResultsForRepo(repo: RepositoryEntity): List<ToolRunEntity> {
--
GitLab