From 1cf3f74eb0e4a76c0ff4255185cea3350efc33a6 Mon Sep 17 00:00:00 2001
From: Tobias Petrasch <tobiaspetrasch@googlemail.com>
Date: Fri, 2 Jun 2023 12:13:34 +0200
Subject: [PATCH] move gitlab configuration to TaskManager and add collection
of repository details as new task
---
.../iem/dataprovider/gitlab/GitlabService.kt | 2 +-
.../dataprovider/taskManager/TaskManager.kt | 17 ++++-
.../dataprovider/taskManager/model/Event.kt | 6 +-
.../tasks/GetRepositoryDetailsTask.kt | 67 +++++++++++++++++++
.../taskManager/tasks/RepositoryDetailsDto.kt | 18 +++++
.../toolRun/RepositoryDetailsRepository.kt | 8 +++
.../dataprovider/toolRun/ToolRunService.kt | 16 +++++
.../toolRun/model/RepositoryDetails.kt | 32 +++++++++
8 files changed, 160 insertions(+), 6 deletions(-)
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GetRepositoryDetailsTask.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/RepositoryDetailsDto.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/RepositoryDetailsRepository.kt
create mode 100644 src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/model/RepositoryDetails.kt
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabService.kt
index f4cad52a..0ced35ee 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/gitlab/GitlabService.kt
@@ -12,7 +12,7 @@ class GitlabService(
) {
suspend fun queryOpenCodeProject(repoId: Long) {
- taskManager.addEvent(RepoChangedEvent(repoId = repoId, gitConfiguration = openCodeGitlabConfiguration))
+ taskManager.addEvent(RepoChangedEvent(repoId = repoId))
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
index 4da97112..d6979699 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/TaskManager.kt
@@ -1,5 +1,6 @@
package de.fraunhofer.iem.dataprovider.taskManager
+import de.fraunhofer.iem.dataprovider.gitlab.OpenCodeGitlabConfiguration
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.taskManager.model.*
import de.fraunhofer.iem.dataprovider.taskManager.tasks.*
@@ -24,8 +25,9 @@ import java.util.*
*/
@Component
class TaskManager(
- private val config: Config,
- private val toolRunService: ToolRunService
+ private val config: Config,
+ private val openCodeGitlabConfiguration: OpenCodeGitlabConfiguration,
+ private val toolRunService: ToolRunService
) {
// The used default dispatcher is ok for CPU-bound workloads. However,
@@ -81,7 +83,7 @@ class TaskManager(
ioWorker.addTask(
GetGitlabProjectTask(
event.repoId,
- event.gitConfiguration,
+ openCodeGitlabConfiguration,
::addEvent,
toolRunService
)
@@ -124,13 +126,22 @@ class TaskManager(
toolRunService
)
+ val gitlabTask = GetRepositoryDetailsTask(
+ event.repoId,
+ openCodeGitlabConfiguration,
+ toolRunService,
+ ::addEvent
+ )
+
taskIds.add(odcTask.taskID)
taskIds.add(detektTask.taskID)
+ taskIds.add(gitlabTask.taskID)
dependentEvents[groupId] = taskIds
worker.addTask(odcTask)
worker.addTask(detektTask)
+ worker.addTask(gitlabTask)
}
is SarifProcessGroupDone -> {
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/model/Event.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/model/Event.kt
index 7d3eaade..83952b82 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/model/Event.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/model/Event.kt
@@ -1,12 +1,12 @@
package de.fraunhofer.iem.dataprovider.taskManager.model
-import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
import de.fraunhofer.iem.dataprovider.taskManager.tasks.GitRepository
+import de.fraunhofer.iem.dataprovider.toolRun.model.RepositoryDetails
import de.fraunhofer.iem.dataprovider.toolRun.model.Sarif
import java.util.*
sealed class Event
-class RepoChangedEvent(val gitConfiguration: GitConfiguration, val repoId: Long) : Event()
+class RepoChangedEvent(val repoId: Long) : Event()
sealed class TaskDone : Event() {
abstract val taskId: UUID
@@ -18,3 +18,5 @@ class GetGitlabProjectDone(override val taskId: UUID, val repoId: UUID, val gitR
class SarifProcessDone(override val taskId: UUID, val repoId: UUID, val sarif: Sarif) : TaskDone()
class SarifProcessGroupDone(override val taskId: UUID, val repoId: UUID, val groupId: UUID, val sarif: Sarif) :
TaskDone()
+
+class GetRepositoryDetailsDone(override val taskId: UUID, val repositoryDetailsEntity: RepositoryDetails) : TaskDone()
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GetRepositoryDetailsTask.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GetRepositoryDetailsTask.kt
new file mode 100644
index 00000000..4fd547b9
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/GetRepositoryDetailsTask.kt
@@ -0,0 +1,67 @@
+package de.fraunhofer.iem.dataprovider.taskManager.tasks
+
+import de.fraunhofer.iem.dataprovider.gitlab.GitConfiguration
+import de.fraunhofer.iem.dataprovider.taskManager.model.Event
+import de.fraunhofer.iem.dataprovider.taskManager.model.GetRepositoryDetailsDone
+import de.fraunhofer.iem.dataprovider.toolRun.ToolRunService
+import de.fraunhofer.iem.dataprovider.toolRun.model.Repository
+import org.gitlab4j.api.GitLabApi
+import org.gitlab4j.api.models.Commit
+import org.gitlab4j.api.models.Project
+import java.util.UUID
+
+class GetRepositoryDetailsTask(
+ private val repoId: UUID,
+ private val gitlabConfiguration: GitConfiguration,
+ private val toolRunService: ToolRunService,
+ override val responseChannel: suspend (event: Event) -> Unit,
+) : Task() {
+ private val gitlabApi: GitLabApi = GitLabApi(gitlabConfiguration.host, gitlabConfiguration.accessToken)
+
+ override suspend fun execute() {
+ logger.info("Gitlab config ${gitlabConfiguration.host} ${gitlabConfiguration.accessToken}")
+ val repositoryEntity = toolRunService.findRepoByID(repoId);
+ logger.info(repositoryEntity.toString())
+ if (repositoryEntity != null) {
+ logger.info("ID: ${repositoryEntity.repoId}")
+ val project = gitlabApi.projectApi.getProject(repositoryEntity.repoId)
+ // Note: We only take commits from the default branch
+ val commits = gitlabApi.commitsApi.getCommits(project.id, project.defaultBranch, ".")
+
+ val numberOfCommits = commits.count()
+ val numberOfSignedCommits = getNumberOfSignedCommits(repositoryEntity, commits)
+ val isDefaultBranchProtected = isDefaultBranchProtected(repositoryEntity, project)
+ val repositoryDetailsDto = RepositoryDetailsDto(repositoryEntity, numberOfCommits, numberOfSignedCommits, isDefaultBranchProtected)
+ logger.info("Collected repository details from $repoId (${repositoryDetailsDto}) successfully")
+ val repositoryDetailsEntity = toolRunService.createRepositoryDetails(repositoryDetailsDto);
+ logger.info("Stored repository details for $repoId (${repositoryDetailsEntity}) successfully")
+ responseChannel(GetRepositoryDetailsDone(taskID, repositoryDetailsEntity))
+ } else {
+ logger.error("Repository $repoId can not be found in the database")
+ }
+ }
+
+ // TODO: This should probably live somewhere else and encapsulate the logic
+ private fun getNumberOfSignedCommits(repositoryEntity: Repository, commits: List<Commit>): Int {
+ var numberOfSignedCommits = 0
+ for (commit in commits) {
+ val signature = gitlabApi.commitsApi.getOptionalGpgSignature(repositoryEntity.repoId, commit.id)
+ if (signature !== null) {
+ numberOfSignedCommits++
+ }
+ }
+ return numberOfSignedCommits
+ }
+
+ // TODO: This should probably live somewhere else and encapsulate the logic
+ private fun isDefaultBranchProtected(repositoryEntity: Repository, project: Project): Boolean {
+ return try {
+ val defaultBranchName = project.defaultBranch;
+ val branch = gitlabApi.repositoryApi.getBranch(repositoryEntity.repoId, defaultBranchName);
+ branch.protected
+ } catch (e: Exception) {
+ // in theory, error probably happens if branch can't be found. In this case we default to false
+ false;
+ }
+ }
+}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/RepositoryDetailsDto.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/RepositoryDetailsDto.kt
new file mode 100644
index 00000000..4c24fec6
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/taskManager/tasks/RepositoryDetailsDto.kt
@@ -0,0 +1,18 @@
+package de.fraunhofer.iem.dataprovider.taskManager.tasks
+
+import de.fraunhofer.iem.dataprovider.toolRun.model.Repository
+import de.fraunhofer.iem.dataprovider.toolRun.model.RepositoryDetails
+import java.sql.Timestamp
+import java.time.Instant
+
+data class RepositoryDetailsDto(val repository: Repository, val numberOfCommits: Int, val numberOfSignedCommits: Int, val isDefaultBranchProtected: Boolean)
+
+fun RepositoryDetailsDto.toDbObject(): RepositoryDetails {
+ val repositoryDetails = RepositoryDetails()
+ repositoryDetails.repository = this.repository
+ repositoryDetails.timestamp = Timestamp.from(Instant.now())
+ repositoryDetails.numberOfCommits = this.numberOfCommits
+ repositoryDetails.numberOfSignedCommits = this.numberOfSignedCommits
+ repositoryDetails.isDefaultBranchProtected = this.isDefaultBranchProtected
+ return repositoryDetails
+}
\ No newline at end of file
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/RepositoryDetailsRepository.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/RepositoryDetailsRepository.kt
new file mode 100644
index 00000000..3d51a9d0
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/RepositoryDetailsRepository.kt
@@ -0,0 +1,8 @@
+package de.fraunhofer.iem.dataprovider.toolRun
+
+import de.fraunhofer.iem.dataprovider.toolRun.model.RepositoryDetails
+import org.springframework.data.jpa.repository.JpaRepository
+import java.util.*
+
+
+interface RepositoryDetailsRepository : JpaRepository<RepositoryDetails, UUID> {}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/ToolRunService.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/ToolRunService.kt
index 2e8808b9..c62e2cbc 100644
--- a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/ToolRunService.kt
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/ToolRunService.kt
@@ -3,6 +3,7 @@ package de.fraunhofer.iem.dataprovider.toolRun
import de.fraunhofer.iem.dataprovider.gitlab.Platform
import de.fraunhofer.iem.dataprovider.logger.getLogger
import de.fraunhofer.iem.dataprovider.taskManager.tasks.GitRepository
+import de.fraunhofer.iem.dataprovider.taskManager.tasks.RepositoryDetailsDto
import de.fraunhofer.iem.dataprovider.taskManager.tasks.toDbObject
import de.fraunhofer.iem.dataprovider.toolRun.model.*
import org.springframework.stereotype.Service
@@ -11,6 +12,7 @@ import java.util.*
@Service
class ToolRunService(
private val repositoryRepository: RepositoryRepository,
+ private val repositoryDetailsRepository: RepositoryDetailsRepository,
private val toolRunRepository: ToolRunRepository,
private val toolRepository: ToolRepository,
private val ruleRepository: RuleRepository
@@ -121,4 +123,18 @@ class ToolRunService(
}
return repo
}
+
+ // TODO: We should decide if we want to return optionals (null if not found) or throw exceptions or ...
+ // TODO: Must be consistent across all methods
+ fun findRepoByID(id: UUID): Repository? {
+ val repository = repositoryRepository.findById(id)
+ if (repository.isEmpty) {
+ return null
+ }
+ return repository.get()
+ }
+
+ fun createRepositoryDetails(repositoryDetailsDto: RepositoryDetailsDto): RepositoryDetails {
+ return repositoryDetailsRepository.save(repositoryDetailsDto.toDbObject())
+ }
}
diff --git a/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/model/RepositoryDetails.kt b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/model/RepositoryDetails.kt
new file mode 100644
index 00000000..52a0fe37
--- /dev/null
+++ b/src/main/kotlin/de/fraunhofer/iem/dataprovider/toolRun/model/RepositoryDetails.kt
@@ -0,0 +1,32 @@
+package de.fraunhofer.iem.dataprovider.toolRun.model
+
+import jakarta.persistence.*
+import org.hibernate.annotations.JdbcTypeCode
+import org.hibernate.type.SqlTypes
+import java.sql.Timestamp
+import java.util.*
+
+@Entity
+@Table(name = "repository_details")
+class RepositoryDetails {
+ @Id
+ @GeneratedValue(strategy = GenerationType.AUTO)
+ @Column(name = "id", nullable = false)
+ var id: UUID? = null
+
+ @ManyToOne(fetch = FetchType.EAGER)
+ var repository: Repository? = null
+
+ @Column(name = "number_of_commits")
+ var numberOfCommits: Int? = null
+
+ @Column(name = "number_of_signed_commits")
+ var numberOfSignedCommits: Int? = null
+
+ @Column(name = "is_default_branch_protected")
+ var isDefaultBranchProtected: Boolean? = null
+
+ @JdbcTypeCode(SqlTypes.TIMESTAMP)
+ @Column(name = "timestamp")
+ var timestamp: Timestamp? = null
+}
\ No newline at end of file
--
GitLab