diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000000000000000000000000000000000000..958acdc54af5960769d83d7df75b979c40d1bca4
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,12 @@
+.env
+docker-compose.yml
+
+# from .gitignore
+.gradle
+build
+.idea
+bin
+**/.env
+**/.DS_Store
+src/test/testResults/*
+tools/db/cockroach-data
diff --git a/.env b/.env
new file mode 100644
index 0000000000000000000000000000000000000000..6c10d6a48acd21f5a9859b829f23395227b9bdfe
--- /dev/null
+++ b/.env
@@ -0,0 +1,10 @@
+# exclude from git with:
+# git update-index --skip-worktree .env
+# revert:
+# git update-index --no-skip-worktree .env
+#
+# e.g. oc000... from https://keycloak.opencode.de/auth/realms/osr/account/#/personal-info
+SECRET_OC_GL_USER=
+# https://gitlab.opencode.de/-/user_settings/personal_access_tokens
+# with read_api, read_user, read_repository, read_registry
+SECRET_OC_GL_APIKEY=
diff --git a/Dockerfile b/Dockerfile
index a41a1d2c38df9ec4643c4f20c72e604954c34748..3b0a185187afbca3a6321e96229660124d97a4eb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,14 +1,17 @@
-FROM eclipse-temurin:21-jdk AS build
+FROM eclipse-temurin:21-jdk-noble AS build
ARG profile=prod
ENV profileEnv=$profile
WORKDIR /app
COPY . /app
-RUN ./gradlew clean build -Dspring.profiles.active=${profile} -x test
-RUN mkdir -p build/dependency && (cd build/dependency; jar -xf ../libs/*-SNAPSHOT.jar)
+RUN set -eux; \
+ ./gradlew clean build -Dspring.profiles.active=${profile} -x test; \
+ mkdir -p build/dependency; \
+ cd build/dependency; \
+ jar -xf ../libs/*-SNAPSHOT.jar
-FROM eclipse-temurin:21-jdk
+FROM eclipse-temurin:21-jdk-noble
ARG profile=prod
ENV profileEnv=$profile
@@ -19,53 +22,48 @@ ENV PYTHONDONTWRITEBYTECODE=1
# Turns off buffering for easier container logging
ENV PYTHONUNBUFFERED=1
+ARG UID=1001
+ARG GID=1002
+
+RUN set -eux; \
+ groupadd --system --gid "$GID" app; \
+ useradd --system --uid "$UID" --gid "$GID" appuser --no-create-home --home /nonexistent
+
# Install dependencies needed to run OCCMD tool
# file is needed by https://github.com/fkie-cad/fact_helper_file
-RUN apt-get update && \
- apt-get install -y --no-install-recommends \
+RUN set -eux; \
+ apt-get update; \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
git \
python3 \
python3-dev \
python3-pip \
+ python3-venv \
libmagic1 \
gcc \
wget \
- file && \
- rm -rf /var/lib/apt/lists/*
-
-WORKDIR /bin
-RUN wget \
- https://github.com/XAMPPRocky/tokei/releases/download/v13.0.0-alpha.0/tokei-x86_64-unknown-linux-musl.tar.gz \
- -O - | \
- tar zxf - && \
- which tokei
+ file; \
+ rm -rf /var/lib/apt/lists/*;
-# The base distro for eclipse-temurin doesn't have the newest pip version
-# which causes weired errors so we upgrade pip.
-RUN pip install --upgrade pip
+WORKDIR /usr/local/bin
+RUN set -eux; \
+ wget https://github.com/XAMPPRocky/tokei/releases/download/v13.0.0-alpha.0/tokei-x86_64-unknown-linux-musl.tar.gz -O - | tar zxf -;\
+ which tokei
WORKDIR /occmd
# Download occmd
-RUN git clone \
- --depth 1 \
- --single-branch --branch main \
- https://gitlab.opencode.de/opencode-analyzer/occmd-public \
- /occmd
-RUN git submodule update --init --recursive
-
-RUN pip3 install -r ./requirements.txt
+RUN set -eux; \
+ git clone --depth 1 --single-branch --branch main https://gitlab.opencode.de/opencode-analyzer/occmd-public.git /occmd; \
+ git submodule update --init --recursive; \
+ python3 -m venv venv; \
+ . venv/bin/activate; \
+ python3 -m pip install wheel; \
+ python3 -m pip install -r requirements.txt; \
+ mkdir -p /occmd/resources/checks/checked_in_binaries/blacklist; \
+ git clone --depth 1 https://gitlab.opencode.de/opencode-analyzer/occmd-checked_in_binaries-blacklist /app/resources/checks/checked_in_binaries/blacklist;
COPY tools/occmd/occmdcfg.ini .
-# Optional: avoid cloning blacklist on each run
-WORKDIR /occmd/resources/checks/checked_in_binaries/blacklist
-RUN git clone \
---depth 1 \
-https://gitlab.opencode.de/opencode-analyzer/occmd-checked_in_binaries-blacklist \
-/app/resources/checks/checked_in_binaries/blacklist
-
-WORKDIR /app
-RUN mkdir -p /app/git
VOLUME /tmp
ARG DEPENDENCY=/app/build/dependency
@@ -74,15 +72,13 @@ COPY --from=build ${DEPENDENCY}/META-INF /app/META-INF
COPY --from=build ${DEPENDENCY}/BOOT-INF/classes /app
# User creation so we don't need to run the image with the root user
-ARG UID=1001
-ARG GID=1002
-
-RUN addgroup --system --gid $GID app \
- && adduser --system --uid $UID --gid $GID appuser --no-create-home --home /nonexistent
-RUN chown -R appuser:app /occmd /app
+WORKDIR /app
+RUN set -eux; \
+ mkdir -p /app/git; \
+ chown -R appuser:app /occmd /app; \
+ chmod u+x /app/scripts/occmd.sh
# Workaround until OCCMD tool is included into CI/CD
-RUN chmod u+x /app/scripts/occmd.sh
USER $UID
ENTRYPOINT ["java","-cp","/app:/app/lib/*", "-Dspring.profiles.active=${profileEnv}", "de.fraunhofer.iem.dataprovider.DataProviderApplicationKt"]
diff --git a/README.md b/README.md
index c4027497c93d8d8ba7ae87cd304c407c49987b4b..6c2f620452084781591aebecf219a47d91aeb9d5 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,23 @@
## Dev Setup
+We have a recommended folder structure for developers, some scripts expecting it:
+
+```
+opencode
+├── data-provider (this repo)
+└── dashboard
+```
+
+1. [add ssh key for authentification and commit signing](https://gitlab.opencode.de/-/user_settings/ssh_keys)
+2. install Docker & test if "docker compose" (v2) is working. [docker-compose is v1 which does currently work but is not recommended](https://docs.docker.com/compose/migrate/#docker-compose-vs-docker-compose)
+ - its suggested to install an alias for compose `printf "\nalias compose='%s'\n" "docker compose" >> ~/.bash_aliases` reload with `source ~/.bash_aliases`
+3. read and fill [.env](./.env)
+4. exclude env from git index via `git update-index --skip-worktree .env`
+5. `docker compose up` which starts opencode-db-1 and data-provider
+
+## Dev Setup old
+
1. Start the database docker service in `tools/db`.
2. Install [OCCMD](https://gitlab.opencode.de/opencode-analyzer/occmd-public.git) and edit the
script `resources/scripts/occmd.sh` to point at the executable.
@@ -16,7 +33,7 @@ opencode.access-token=${OC_GL_APIKEY} // this is the api key for opencode
// either connect to a local postgresql db or the deployed test db
// the test db is a cockroach db and the expected url schema looks as follows:
// #spring.datasource.url=jdbc:postgresql://${host}:26257/${DB_USER}?sslmode=${ssl_mode}&sslrootcert=${ca_crt}&sslcert=${ssl_cert}&sslkey=${ssl_key}
-spring.datasource.url=${DB_URL}
+spring.datasource.url=${DB_URL}
spring.datasource.username=${DB_USER}
spring.datasource.password=${DB_PW}
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000000000000000000000000000000000000..77bb825832c3587df2af9201d2672aff2004ac6b
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,38 @@
+name: opencode
+
+include:
+ - path: ./tools/db/docker-compose.yml
+
+services:
+ data-provider:
+ depends_on:
+ db:
+ condition: service_healthy
+ required: true
+ image: data-provider
+ build:
+ context: .
+ args:
+ profile: dev
+ environment:
+ - OCCMD_PATH=/app/scripts/occmd.sh
+ - OC_GL_USER=${SECRET_OC_GL_USER}
+ - OC_GL_APIKEY=${SECRET_OC_GL_APIKEY}
+ - XDG_CONFIG_HOME=/app/.config/
+ - GIT_CLONE_TARGET_DIRECTORY=/app/git/
+ - USE_MANAGEMENT_PORT=false # Needs to define a SwaggerWelcomeCommon bean if true
+ - PROJECT_IDS=1448,2991,1317,560,2188,2155,2149,2235 # occmd-public, Covid19 fraud detection, Opendesk, Helm chart, e2e tests
+ - CORS_ORIGIN=* # Must be without quotes i.e. allow everything: * | originally https://sec-kpi.opencode.de
+
+ - PORT=4000
+ - DB_HOST=db
+ - DB_USER=sa
+ - DB_NAME=dataprovider
+ - DB_PORT=26257
+ - HMAC_KEY=123456789012345678901234567890
+
+ - ADMIN_PASSWORD=01234567890123456789
+ - ADMIN_USERNAME=012345678901234567890123456789
+ - API_KEY=012345678901234567890123456789
+ ports:
+ - 4000:4000
diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties
index 151cb80241d70478a6c3c73407aaac31dffa7e63..c2571ead0905f2f5dfe10a9fcd4a2142fc41471d 100644
--- a/src/main/resources/application-dev.properties
+++ b/src/main/resources/application-dev.properties
@@ -28,7 +28,7 @@ occmd.git-clone-target-directory=${GIT_CLONE_TARGET_DIRECTORY:}
occmd.occmd-path=${OCCMD_PATH:}
server.port=${PORT}
management.server.port=${MANAGEMENT_PORT:}
-springdoc.use-management-port=true
+springdoc.use-management-port=${USE_MANAGEMENT_PORT:}
management.endpoints.web.exposure.include=health,metrics, openapi, swagger-ui, logfile, loggers, prometheus, auditevents
springdoc.show-actuator=true
spring.main.web-application-type=REACTIVE
@@ -44,7 +44,7 @@ spring.jpa.open-in-view=false
spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false
spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.CockroachDialect
# DB Login data
-spring.datasource.url=jdbc:postgresql://${host}:26257/${DB_USER}?sslmode=${ssl_mode}&sslrootcert=${ca_crt}&sslcert=${ssl_cert}&sslkey=${ssl_key}
+spring.datasource.url=jdbc:postgresql://${DB_HOST:}:${DB_PORT:}/${DB_NAME:}?sslmode=disable&user=root
spring.datasource.username=${DB_USER:}
spring.datasource.driver-class-name=org.postgresql.Driver
-projects.project-ids=159, 106, 124
+projects.project-ids=${PROJECT_IDS:}
diff --git a/src/main/resources/scripts/occmd.sh b/src/main/resources/scripts/occmd.sh
index 7395cd7e81b07af70c734682820f17fdc18ae89b..a9624b262beb0ef7281813db80836273de9a294a 100755
--- a/src/main/resources/scripts/occmd.sh
+++ b/src/main/resources/scripts/occmd.sh
@@ -1,12 +1,11 @@
-#!/bin/sh
+#!/bin/bash
+
+set -euo pipefail
INSTALL_DIR=/occmd
PROJ_PATH=${1}
PROJ_ID=${2}
-USER_NAME=${3}
-API_KEY=${4}
-URL=${5}
-
-export OC_GL_APIKEY="${API_KEY}" && export OC_GL_USER="${USER_NAME}" && export OC_GL_URL="${URL}" && cd "${INSTALL_DIR}" && ./occmd check -d "${PROJ_PATH}" -i "${PROJ_ID}"
-
-exit 0
+cd "${INSTALL_DIR}"
+#shellcheck disable=SC1091
+source venv/bin/activate
+./occmd check -d "${PROJ_PATH}" -i "${PROJ_ID}"
diff --git a/tools/db/docker-compose.yml b/tools/db/docker-compose.yml
index 66c87ed43e4f4b57131aafb06943d114e799747c..743d19a9997704aa70243aa23a7e99c8ecb436ef 100644
--- a/tools/db/docker-compose.yml
+++ b/tools/db/docker-compose.yml
@@ -1,32 +1,25 @@
+name: opencode
+
+volumes:
+ database:
services:
- crdb:
+ db:
image: cockroachdb/cockroach:v24.1.2
restart: always
environment:
- COCKROACH_USER=sa
- - COCKROACH_PASSWORD=password
- COCKROACH_DATABASE=dataprovider
ports:
- "26257:26257"
- "8083:8080"
command: start-single-node --insecure
volumes:
- - "${PWD}/cockroach-data/crdb:/cockroach/cockroach-data"
-
-# db:
-# image: postgres:latest
-# restart: always
-# environment:
-# - POSTGRES_USER=sa
-# - POSTGRES_PASSWORD=password
-# - POSTGRES_DB=dataprovider
-# ports:
-# - "5432:5432"
-# api:
-# build: .
-# restart: always
-# ports:
-# - "3000:3000"
-# command: json-server --watch api.json
-# volumes:
-# - "./api.json:/api.json"
+ - "database:/cockroach/cockroach-data"
+ healthcheck:
+ # user / database is taken from env variables
+ # cockroach creates at the end the database + user
+ test: ["CMD", "cockroach", "sql", "--insecure", "-e", "SELECT version()"]
+ interval: 5s
+ timeout: 5s
+ retries: 5
+ start_period: 30s
diff --git a/tools/occmd/occmdcfg.ini b/tools/occmd/occmdcfg.ini
index 97233a259bb02b57a81d39ae4836fce7a608a2c5..fe290ce0f1bddf223e647ebff9256d8c7871278e 100644
--- a/tools/occmd/occmdcfg.ini
+++ b/tools/occmd/occmdcfg.ini
@@ -31,4 +31,4 @@ pl_loc_timeout = 60
pl_loc_ncpu = 4
[oc]
-gl_url =
+gl_url = https://gitlab.opencode.de