From 7a3d9f4287b6c5dcc91e209d6a668e398e27c85a Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 11 Jul 2023 16:38:44 +0200 Subject: [PATCH 1/9] [CTX-16] fix: don't use Elasticsearch scroll to find user by its API key --- app/org/thp/cortex/services/KeyAuthSrv.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/org/thp/cortex/services/KeyAuthSrv.scala b/app/org/thp/cortex/services/KeyAuthSrv.scala index 62eb29417..527170e98 100644 --- a/app/org/thp/cortex/services/KeyAuthSrv.scala +++ b/app/org/thp/cortex/services/KeyAuthSrv.scala @@ -29,9 +29,8 @@ class KeyAuthSrv @Inject() (userSrv: UserSrv, implicit val ec: ExecutionContext, import org.elastic4play.services.QueryDSL._ // key attribute is sensitive so it is not possible to search on that field userSrv - .find("status" ~= "Ok", Some("all"), Nil) + .find(and("status" ~= "Ok", "key" ~= key), Some("0-1"), Nil) ._1 - .filter(_.key().contains(key)) .runWith(Sink.headOption) .flatMap { case Some(user) => userSrv.getFromUser(request, user, name) From 39fd894d76efbf09cad3dd02a254757edc24f3fa Mon Sep 17 00:00:00 2001 From: toom Date: Thu, 27 Jul 2023 17:29:31 +0200 Subject: [PATCH 2/9] [CTX-17] fix: Update version of JFFI (#448) --- build.sbt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.sbt b/build.sbt index cbc06ead0..bfa0e4367 100644 --- a/build.sbt +++ b/build.sbt @@ -23,6 +23,12 @@ lazy val cortex = (project in file(".")) Dependencies.akkaCluster, Dependencies.akkaClusterTyped ), + dependencyOverrides ++= Seq( + "com.github.jnr" % "jffi" % "1.3.11", + "com.github.jnr" % "jnr-ffi" % "2.2.13", + "com.github.jnr" % "jnr-enxio" % "0.32.14", + "com.github.jnr" % "jnr-unixsocket" % "0.38.19" + ), resolvers += Resolver.sbtPluginRepo("releases"), resolvers += "scalaz-bintray" at "https://dl.bintray.com/scalaz/releases", resolvers += "elasticsearch-releases" at "https://artifacts.elastic.co/maven", From 49daf371e92999af7b89fcaa5de927a8b9db5c97 Mon Sep 17 00:00:00 2001 From: Vincent Debergue Date: Thu, 27 Jul 2023 17:29:59 +0200 Subject: [PATCH 3/9] Update deps (#449) * ci: add github workflow * update server dependencies * update docker image * create a github release * update node version --- .github/workflows/build.yml | 115 +++++++++++++++++++++++++++++++ .github/workflows/check_code.yml | 22 ++++++ build.sbt | 9 +++ project/Dependencies.scala | 21 +++--- project/DockerSettings.scala | 105 +++++++++++++++------------- project/FrontEnd.scala | 2 +- project/build.properties | 2 +- project/plugins.sbt | 4 +- www/package.json | 7 +- 9 files changed, 224 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/check_code.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..5c4ce6463 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,115 @@ +name: Build +on: + push: + tags: ["*"] + +jobs: + build: + name: Build + runs-on: [ ubuntu-latest ] + steps: + - uses: actions/checkout@v3 + - name: Setup node + uses: actions/setup-node@v3 + timeout-minutes: 15 + continue-on-error: true + with: + node-version: 18.16 + - name: Install bower + run: npm install -g bower + - name: Import PGP Key + run: gpg --batch --import - <<< ${{ secrets.PGP_KEY }} + - name: Write version + id: version + run: | + V=$(sbt -no-colors --error "print version" | awk 'END{print $1}') + echo "version=$V" + echo "version=$V" >> $GITHUB_OUTPUT + - name: Build packages + run: sbt Docker/stage Debian/packageBin Rpm/packageBin Universal/packageBin cortexWithDeps/Docker/stage makeBom + - name: Move packages + run: | + mv target/rpm/RPMS/noarch/cortex*.rpm target/ + mv target/universal/cortex*.zip target/ + + - name: Write docker tags from version + id: tags + run: | + V=${{ steps.version.outputs.version }} + if ( echo $V | grep -qi rc ) + then + echo $( echo $V | sed -re 's/([0-9]+.[0-9]+.[0-9]+)-RC([0-9]+)-([0-9]+)/\1-RC\2,\1-RC\2-\3/' ) > target/tags + else + echo $( echo $V | sed -re 's/([0-9]+).([0-9]+).([0-9]+)-([0-9]+)/\1,\1.\2,\1.\2.\3,\1.\2.\3-\4,latest/' ) > target/tags + fi + echo "tags=$(cat target/tags)" >> $GITHUB_OUTPUT + + - name: Build list of additional tags + id: additional-tags + uses: actions/github-script@v6 + with: + script: | + core.setOutput('tags', `${{ steps.tags.outputs.tags }}`.split(",").join("\n")) + + - name: Generate full docker tags + id: meta + uses: docker/metadata-action@v4 + with: + images: | + name=${{ secrets.HARBOR_REGISTRY }}/thehiveproject/cortex + name=thehiveproject/cortex + tags: | + ${{ steps.additional-tags.outputs.tags }} + + - name: Login to Harbor + uses: docker/login-action@v2 + with: + registry: ${{ secrets.HARBOR_REGISTRY }} + username: ${{ secrets.HARBOR_USERNAME }} + password: ${{ secrets.HARBOR_PASSWORD }} + + - name: Login to Dockerhub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push docker + uses: docker/build-push-action@v3 + with: + context: target/docker/stage + push: true + tags: ${{ steps.meta.outputs.tags }} + + - name: Build and push fat docker + uses: docker/build-push-action@v3 + with: + context: target/docker-withdeps/target/docker/stage + push: true + tags: ${{ steps.meta.outputs.tags }} + + - name: Make github release + uses: "softprops/action-gh-release@v1" + id: make-release + with: + generate_release_notes: true + files: | + target/cortex*.deb + target/cortex*.rpm + target/cortex*.zip + target/cortex*.bom.xml + + notify: + needs: [ build ] + runs-on: [ ubuntu-latest ] + if: always() + steps: + - name: Slack notification + uses: Gamesight/slack-workflow-status@master + with: + repo_token: ${{secrets.GITHUB_TOKEN}} + slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}} + channel: "#ci-cortex" + name: Cortex build + include_commit_message: true + include_jobs: true \ No newline at end of file diff --git a/.github/workflows/check_code.yml b/.github/workflows/check_code.yml new file mode 100644 index 000000000..f3aa44ffd --- /dev/null +++ b/.github/workflows/check_code.yml @@ -0,0 +1,22 @@ +name: Check Code +on: + workflow_dispatch: + workflow_call: + push: + branches: [master, develop] +jobs: + check: + name: Check + runs-on: [ ubuntu-latest ] + steps: + - uses: actions/checkout@v3 + - name: Setup node + uses: actions/setup-node@v3 + timeout-minutes: 15 + continue-on-error: true + with: + node-version: 18.16 + - name: Install bower + run: npm install -g bower + - name: Run tests + run: sbt test Universal/packageBin diff --git a/build.sbt b/build.sbt index bfa0e4367..94b60e4ae 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,14 @@ import Common._ +ThisBuild / scalaVersion := Dependencies.scalaVersion +ThisBuild / evictionErrorLevel := util.Level.Warn + +ThisBuild / dependencyOverrides ++= Seq( + Dependencies.Play.twirl, + "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.5", + "org.apache.commons" % "commons-compress" % "1.23.0", + "com.google.guava" % "guava" % "32.1.1-jre" +) lazy val cortex = (project in file(".")) .enablePlugins(PlayScala) .settings(projectSettings) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1e4701281..7d07b526f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,21 +5,22 @@ object Dependencies { object Play { val version = play.core.PlayVersion.current - val ws = "com.typesafe.play" %% "play-ws" % version - val ahc = "com.typesafe.play" %% "play-ahc-ws" % version - val cache = "com.typesafe.play" %% "play-ehcache" % version - val test = "com.typesafe.play" %% "play-test" % version - val specs2 = "com.typesafe.play" %% "play-specs2" % version + val ws = "com.typesafe.play" %% "play-ws" % version exclude ("com.typesafe.play", "play-ws-standalone-xml") + val ahc = "com.typesafe.play" %% "play-ahc-ws" % version + val cache = "com.typesafe.play" %% "play-ehcache" % version + val test = "com.typesafe.play" %% "play-test" % version + val specs2 = "com.typesafe.play" %% "play-specs2" % version val filters = "com.typesafe.play" %% "filters-helpers" % version - val guice = "com.typesafe.play" %% "play-guice" % version + val guice = "com.typesafe.play" %% "play-guice" % version + val twirl = "com.typesafe.play" %% "twirl-api" % "1.5.2" } - val scalaGuice = "net.codingwell" %% "scala-guice" % "5.1.0" + val scalaGuice = "net.codingwell" %% "scala-guice" % "5.1.1" - val reflections = "org.reflections" % "reflections" % "0.10.2" - val zip4j = "net.lingala.zip4j" % "zip4j" % "2.10.0" + val reflections = "org.reflections" % "reflections" % "0.10.2" + val zip4j = "net.lingala.zip4j" % "zip4j" % "2.11.5" val elastic4play = "org.thehive-project" %% "elastic4play" % "1.13.6" - val dockerClient = "com.spotify" % "docker-client" % "8.14.4" + val dockerClient = "com.spotify" % "docker-client" % "8.16.0" val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % play.core.PlayVersion.akkaVersion val akkaClusterTyped = "com.typesafe.akka" %% "akka-cluster-typed" % play.core.PlayVersion.akkaVersion } diff --git a/project/DockerSettings.scala b/project/DockerSettings.scala index 10f8e26ba..8a0262eb9 100644 --- a/project/DockerSettings.scala +++ b/project/DockerSettings.scala @@ -29,14 +29,21 @@ object DockerSettings { case (_, filepath) => filepath == "/opt/cortex/conf/application.conf" }), dockerCommands := Seq( - Cmd("FROM", "openjdk:8-slim"), + Cmd("FROM", "debian:bullseye-slim"), Cmd("LABEL", "MAINTAINER=\"TheHive Project \"", "repository=\"https://github.com/TheHive-Project/TheHive\""), Cmd("WORKDIR", "/opt/cortex"), + Cmd("ENV", "JAVA_HOME", "/usr/lib/jvm/java-11-amazon-corretto"), // format: off Cmd("RUN", "apt", "update", "&&", "apt", "upgrade", "-y", "&&", - "apt", "install", "-y", "iptables", "lxc", "wget", "&&", + "apt", "install", "-y", "iptables", "lxc", "wget", "curl", "gnupg", "&&", + // install java corretto + "curl", "-fL", "https://apt.corretto.aws/corretto.key", "|", "gpg", "--dearmor", "-o", "/usr/share/keyrings/corretto.gpg", "&&", + "echo", "'deb [signed-by=/usr/share/keyrings/corretto.gpg] https://apt.corretto.aws stable main'", ">", "/etc/apt/sources.list.d/corretto.list", "&&", + "mkdir", "-p", "/usr/share/man/man1", "||", "true", "&&", + "apt", "update", "&&", "apt", "install", "-y", "java-11-amazon-corretto-jdk", "&&", + // setup for docker "apt", "autoclean", "-y", "-q", "&&", "apt", "autoremove", "-y", "-q", "&&", "wget", "-q", "-O", "-", "https://download.docker.com/linux/static/stable/x86_64/docker-18.09.0.tgz", "|", @@ -46,8 +53,10 @@ object DockerSettings { "addgroup", "--system", "docker", "&&", "echo", "dockremap:165536:65536", ">>", "/etc/subuid", "&&", "echo", "dockremap:165536:65536", ">>", "/etc/subgid", "&&", + // cleanup "rm", "-rf", "/var/lib/apt/lists/*", "&&", "(", "type", "groupadd", "1>/dev/null", "2>&1", "&&", + // setup cortex user "groupadd", "-g", "1001", "cortex", "||", "addgroup", "-g", "1001", "-S", "cortex", ")", "&&", @@ -72,52 +81,52 @@ object DockerSettings { Cmd( "RUN", """ - | apt update && - | apt upgrade -y && - | apt install -y -q --no-install-recommends --no-install-suggests - | wkhtmltopdf libfuzzy-dev libimage-exiftool-perl curl unzip - | libboost-regex-dev - | libboost-program-options-dev - | libboost-system-dev libboost-filesystem-dev libssl-dev - | build-essential cmake python3-dev python2-dev - | git python3 python3-pip libffi-dev libjpeg62-turbo-dev libtiff5-dev - | libopenjp2-7-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev - | tcl8.6-dev tk8.6-dev python3-tk libharfbuzz-dev libfribidi-dev - | libxcb1-dev python2.7 && - | rm -rf /var/lib/apt/lists/* && - | curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output /tmp/get-pip.py && - | python2.7 /tmp/get-pip.py && - | pip2 install -U setuptools && - | pip3 install -U setuptools && - | ln -sf python3 /usr/bin/python && - | hash -r && - | git clone https://github.com/JusticeRage/Manalyze.git /tmp/Manalyze && - | cd /tmp/Manalyze && - | cmake . && - | make -j5 && - | cd /tmp/Manalyze/bin/yara_rules && - | pip3 install requests && - | python3 update_clamav_signatures.py && - | cd /tmp/Manalyze && - | make install && - | cd / && - | rm -rf /tmp/Manalyze && - | curl -SL https://github.com/fireeye/flare-floss/releases/download/v1.7.0/floss-v1.7.0-linux.zip - | --output /tmp/floss.zip && - | unzip /tmp/floss.zip -d /usr/bin && - | rm /tmp/floss.zip && - | git clone https://github.com/TheHive-Project/Cortex-Analyzers.git /tmp/analyzers && - | cat $(find /tmp/analyzers -name requirements.txt) | sort -u | while read I ; - | do - | pip2 install $I || true && - | pip3 install $I || true ; - | done && - | for I in $(find /tmp/analyzers -name requirements.txt) ; - | do - | pip2 install -r $I || true && - | pip3 install -r $I || true ; - | done && - | rm -rf /tmp/analyzers + | apt update && + | apt upgrade -y && + | apt install -y -q --no-install-recommends --no-install-suggests + | wkhtmltopdf libfuzzy-dev libimage-exiftool-perl curl unzip + | libboost-regex-dev + | libboost-program-options-dev + | libboost-system-dev libboost-filesystem-dev libssl-dev + | build-essential cmake python3-dev python2-dev + | git python3 python3-pip libffi-dev libjpeg62-turbo-dev libtiff5-dev + | libopenjp2-7-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev + | tcl8.6-dev tk8.6-dev python3-tk libharfbuzz-dev libfribidi-dev + | libxcb1-dev python2.7 && + | rm -rf /var/lib/apt/lists/* && + | curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output /tmp/get-pip.py && + | python2.7 /tmp/get-pip.py && + | pip2 install -U setuptools && + | pip3 install -U setuptools && + | ln -sf python3 /usr/bin/python && + | hash -r && + | git clone https://github.com/JusticeRage/Manalyze.git /tmp/Manalyze && + | cd /tmp/Manalyze && + | cmake . && + | make -j5 && + | cd /tmp/Manalyze/bin/yara_rules && + | pip3 install requests && + | python3 update_clamav_signatures.py && + | cd /tmp/Manalyze && + | make install && + | cd / && + | rm -rf /tmp/Manalyze && + | curl -SL https://github.com/fireeye/flare-floss/releases/download/v1.7.0/floss-v1.7.0-linux.zip + | --output /tmp/floss.zip && + | unzip /tmp/floss.zip -d /usr/bin && + | rm /tmp/floss.zip && + | git clone https://github.com/TheHive-Project/Cortex-Analyzers.git /tmp/analyzers && + | cat $(find /tmp/analyzers -name requirements.txt) | sort -u | while read I ; + | do + | pip2 install $I || true && + | pip3 install $I || true ; + | done && + | for I in $(find /tmp/analyzers -name requirements.txt) ; + | do + | pip2 install -r $I || true && + | pip3 install -r $I || true ; + | done && + | rm -rf /tmp/analyzers """.stripMargin.split("\\s").filter(_.nonEmpty): _* ) ) diff --git a/project/FrontEnd.scala b/project/FrontEnd.scala index de418c40f..294a947aa 100644 --- a/project/FrontEnd.scala +++ b/project/FrontEnd.scala @@ -18,7 +18,7 @@ object FrontEnd extends AutoPlugin { val s = streams.value s.log.info("Building front-end ...") s.log.info("npm install") - Process("npm" :: "install" :: Nil, baseDirectory.value / "www") ! s.log + Process("npm" :: "install" :: "--legacy-peer-deps" :: Nil, baseDirectory.value / "www") ! s.log s.log.info("npm run build") Process("npm" :: "run" :: "build" :: Nil, baseDirectory.value / "www") ! s.log val dir = baseDirectory.value / "www" / "dist" diff --git a/project/build.properties b/project/build.properties index baf5ff3ec..40b3b8e7b 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.7 +sbt.version=1.9.0 diff --git a/project/plugins.sbt b/project/plugins.sbt index 7207b3c38..758cc8da9 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,7 +1,9 @@ // Comment to get more information during initialization logLevel := Level.Info +evictionErrorLevel := util.Level.Warn // The Play plugin -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.16") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.19") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") addSbtPlugin("org.thehive-project" % "sbt-github-changelog" % "0.4.0") +addSbtPlugin("io.github.siculo" %% "sbt-bom" % "0.3.0") diff --git a/www/package.json b/www/package.json index d38b1847a..c5f90d233 100755 --- a/www/package.json +++ b/www/package.json @@ -17,6 +17,8 @@ }, "dependencies": { "@uirouter/angularjs": "^1.0.22", + "@uirouter/core": "^6.1.0", + "@uirouter/rx": "^1.0.0", "angular": "^1.7.8", "angular-base64-upload": "^0.1.23", "angular-bootstrap-multiselect": "git+https://github.com/bentorfs/angular-bootstrap-multiselect.git", @@ -67,12 +69,13 @@ "html-webpack-plugin": "2.22.0", "manifest-revision-webpack-plugin": "0.3.0", "ngtemplate-loader": "^1.3.1", - "node-sass": "^4.12.0", + "node-sass": "npm:sass@^1.62.0", "postcss-loader": "^0.13.0", + "sass": "^1.62.0", "sass-loader": "^4.0.2", "style-loader": "^0.13.1", "url-loader": "^0.5.9", "webpack": "3.5.0", "webpack-dev-server": "2.2.0" } -} +} \ No newline at end of file From e779d6431fbeb45aa8b7fda10b3ea711b7fb8395 Mon Sep 17 00:00:00 2001 From: Vincent Debergue Date: Thu, 14 Sep 2023 11:22:05 +0200 Subject: [PATCH 4/9] update dependencies for rpm and debian packages (#456) --- project/PackageSettings.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/PackageSettings.scala b/project/PackageSettings.scala index a2ed84973..35d365b6d 100644 --- a/project/PackageSettings.scala +++ b/project/PackageSettings.scala @@ -42,7 +42,7 @@ object PackageSettings { rpmVendor := organizationName.value, rpmUrl := organizationHomepage.value.map(_.toString), rpmLicense := Some("AGPL"), - rpmRequirements += "java-1.8.0-openjdk-headless", + rpmRequirements += "java-11-openjdk-headless", Rpm / maintainerScripts := maintainerScriptsFromDirectory( baseDirectory.value / "package" / "rpm", Seq(RpmConstants.Pre, RpmConstants.Preun, RpmConstants.Post, RpmConstants.Postun) @@ -79,7 +79,7 @@ object PackageSettings { } }, debianPackageRecommends := Seq("elasticsearch"), - debianPackageDependencies += "java8-runtime | java8-runtime-headless", + debianPackageDependencies += "default-jre-headless | openjdk-11-jre-headless", Debian / maintainerScripts := maintainerScriptsFromDirectory( baseDirectory.value / "package" / "debian", Seq(DebianConstants.Postinst, DebianConstants.Prerm, DebianConstants.Postrm) From 77495008c58a75c24d2d6eb49be703d3228de126 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 22 Jun 2022 17:57:16 +0200 Subject: [PATCH 5/9] Add SECURITY.md --- SECURITY.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..8fef7c7a8 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,28 @@ +# StrangeBee Security Policies + +At [StrangeBee](https://www.strangebee.com) we take the security our software and services seriously, including following applications and projects: +- TheHive (TheHive 5, and [previous open source version](https://github.com/TheHive-Project/TheHive)) +- [Cortex](https://github.com/TheHive-Project/Cortex) +- [Cortex-Analyzers](https://github.com/TheHive-Project/Cortex-Analyzers) + +## Reporting a vulnerability +If you believe you have found a security vulnerability in our applications and services (TheHive, Cortex, Cortex-Analyzers ...), report it to us. + +**Please do not report security vulnerabilities through public GitHub issues, discussions, or pull requests.** + +Instead, please send security vulnerabilities by emailing the StrangeBee Security team: + +``` +security[@]strangebee.com +``` + +In this email, please include as much information as possible that can help us better understand and resolve the issue: +- Application and version +- Special configuration and usage required to reproduce the issue +- Step-by-step instructions to reproduce the issue +- Exploit code is any +- Impact of the issue + +This will be very useful and help us triage your report more quickly. + +More information regarding our Security policies and Advisories can be found here: [https://github.com/StrangeBeeCorp/security](). From f7fdcdf4325bb3b790840fc0813c064149566b9c Mon Sep 17 00:00:00 2001 From: Jiri Prochazka Date: Thu, 1 Dec 2022 10:22:06 +0100 Subject: [PATCH 6/9] GroupUserMapper.scala: Backport fix from TheHive for group mapper functionality This commit fixes issue #344 by backporting fix from TheHive repository. Original pull request from which the backport was taken can be found at https://github.com/TheHive-Project/TheHive/pull/1112. --- .../services/mappers/GroupUserMapper.scala | 83 ++++++++++++++++--- 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/app/org/thp/cortex/services/mappers/GroupUserMapper.scala b/app/org/thp/cortex/services/mappers/GroupUserMapper.scala index 5768d5b30..ab50f1c6b 100644 --- a/app/org/thp/cortex/services/mappers/GroupUserMapper.scala +++ b/app/org/thp/cortex/services/mappers/GroupUserMapper.scala @@ -1,24 +1,25 @@ package org.thp.cortex.services.mappers import scala.concurrent.{ExecutionContext, Future} +import scala.util.parsing.combinator._ -import play.api.Configuration +import play.api.{Configuration, Logger} import play.api.libs.json._ import play.api.libs.ws.WSClient import javax.inject.Inject -import org.elastic4play.AuthenticationError +import org.elastic4play.{AuthenticationError, AuthorizationError} import org.elastic4play.controllers.Fields class GroupUserMapper( loginAttrName: String, nameAttrName: String, - groupAttrName: String, + groupsAttrName: String, organizationAttrName: Option[String], defaultRoles: Seq[String], defaultOrganization: Option[String], - groupsUrl: String, + groupsUrl: Option[String], mappings: Map[String, Seq[String]], ws: WSClient, implicit val ec: ExecutionContext @@ -32,7 +33,7 @@ class GroupUserMapper( configuration.getOptional[String]("auth.sso.attributes.organization"), configuration.getOptional[Seq[String]]("auth.sso.defaultRoles").getOrElse(Seq()), configuration.getOptional[String]("auth.sso.defaultOrganization"), - configuration.getOptional[String]("auth.sso.groups.url").getOrElse(""), + configuration.getOptional[String]("auth.sso.groups.url"), configuration.getOptional[Map[String, Seq[String]]]("auth.sso.groups.mappings").getOrElse(Map()), ws, ec @@ -40,13 +41,73 @@ class GroupUserMapper( override val name: String = "group" + private[GroupUserMapper] lazy val logger = Logger(getClass) + + private class RoleListParser extends RegexParsers { + val str = "[a-zA-Z0-9_]+".r + val strSpc = "[a-zA-Z0-9_ ]+".r + val realStr = ("\""~>strSpc<~"\"" | "'"~>strSpc<~"'" | str) + + def expr: Parser[Seq[String]] = { + "[" ~ opt(realStr ~ rep("," ~ realStr)) ~ "]" ^^ { + case _ ~ Some(firstRole ~ list) ~ _ ⇒ list.foldLeft(Seq(firstRole)) { + case (queue, _ ~ role) ⇒ role +: queue + } + case _ ~ _ ⇒ Seq.empty[String] + } | opt(realStr) ^^ { + case Some(role) ⇒ Seq(role) + case None ⇒ Seq.empty[String] + } + } + } + override def getUserFields(jsValue: JsValue, authHeader: Option[(String, String)]): Future[Fields] = { - val apiCall = authHeader.fold(ws.url(groupsUrl))(headers => ws.url(groupsUrl).addHttpHeaders(headers)) - apiCall.get.flatMap { r => - val jsonGroups = (r.json \ groupAttrName).as[Seq[String]] - val mappedRoles = jsonGroups.flatMap(mappings.get).maxBy(_.length) - val roles = if (mappedRoles.nonEmpty) mappedRoles else defaultRoles + groupsUrl match { + case Some(groupsEndpointUrl) ⇒ { + logger.debug(s"Retreiving groups from ${groupsEndpointUrl}") + val apiCall = authHeader.fold(ws.url(groupsEndpointUrl))(headers ⇒ ws.url(groupsEndpointUrl).addHttpHeaders(headers)) + apiCall.get.flatMap { r ⇒ extractGroupsThenBuildUserFields(jsValue, r.json) } + } + case None ⇒ { + logger.debug(s"Extracting groups from user info") + extractGroupsThenBuildUserFields(jsValue, jsValue) + } + } + } + + private def extractGroupsThenBuildUserFields(jsValue: JsValue, groupsContainer: JsValue): Future[Fields] = { + (groupsContainer \ groupsAttrName) match { + // Groups received as valid JSON array + case JsDefined(JsArray(groupsList)) ⇒ mapGroupsAndBuildUserFields(jsValue, groupsList.map(_.as[String]).toList) + + // Groups list received as string (invalid JSON, for example: "ROLE" or "['Role 1', ROLE2, 'Role_3']") + case JsDefined(JsString(groupsStr)) ⇒ { + val parser = new RoleListParser + parser.parseAll(parser.expr, groupsStr) match { + case parser.Success(result, _) ⇒ mapGroupsAndBuildUserFields(jsValue, result) + case err: parser.NoSuccess ⇒ Future.failed(AuthenticationError(s"User info fails: can't parse groups list (${err.msg})")) + } + } + + // Invalid group list + case JsDefined(error) ⇒ + Future.failed(AuthenticationError(s"User info fails: invalid groups list received in user info ('${error}' of type ${error.getClass})")) + + // Groups field is undefined + case _: JsUndefined ⇒ + Future.failed(AuthenticationError(s"User info fails: groups attribute ${groupsAttrName} doesn't exist in user info")) + } + } + + private def mapGroupsAndBuildUserFields(jsValue: JsValue, jsonGroups: Seq[String]): Future[Fields] = { + val mappedRoles = jsonGroups.flatMap(mappings.get).flatten.toSet + val roles = if (mappedRoles.nonEmpty) mappedRoles else defaultRoles + + if (roles.isEmpty) { + Future.failed(AuthorizationError(s"No matched roles for user")) + } else { + logger.debug(s"Computed roles: ${roles.mkString(", ")}") val fields = for { login <- (jsValue \ loginAttrName).validate[String] @@ -58,7 +119,7 @@ class GroupUserMapper( } yield Fields(Json.obj("login" -> login.toLowerCase, "name" -> name, "roles" -> roles, "organization" -> organization)) fields match { case JsSuccess(f, _) => Future.successful(f) - case JsError(errors) => Future.failed(AuthenticationError(s"User info fails: ${errors.map(_._1).mkString}")) + case JsError(errors) ⇒ Future.failed(AuthenticationError(s"User info fails: ${errors.map(_._2).map(_.map(_.messages.mkString(", ")).mkString("; ")).mkString}")) } } } From 5da91c88e6e3bc472efd6d1e0ffd12fd5e4e7d55 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 20 Sep 2023 14:15:07 +0200 Subject: [PATCH 7/9] Reformat code --- .../services/mappers/GroupUserMapper.scala | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/app/org/thp/cortex/services/mappers/GroupUserMapper.scala b/app/org/thp/cortex/services/mappers/GroupUserMapper.scala index ab50f1c6b..c276a6001 100644 --- a/app/org/thp/cortex/services/mappers/GroupUserMapper.scala +++ b/app/org/thp/cortex/services/mappers/GroupUserMapper.scala @@ -2,16 +2,16 @@ package org.thp.cortex.services.mappers import scala.concurrent.{ExecutionContext, Future} import scala.util.parsing.combinator._ - import play.api.{Configuration, Logger} import play.api.libs.json._ import play.api.libs.ws.WSClient import javax.inject.Inject - import org.elastic4play.{AuthenticationError, AuthorizationError} import org.elastic4play.controllers.Fields +import scala.util.matching.Regex + class GroupUserMapper( loginAttrName: String, nameAttrName: String, @@ -44,61 +44,57 @@ class GroupUserMapper( private[GroupUserMapper] lazy val logger = Logger(getClass) private class RoleListParser extends RegexParsers { - val str = "[a-zA-Z0-9_]+".r - val strSpc = "[a-zA-Z0-9_ ]+".r - val realStr = ("\""~>strSpc<~"\"" | "'"~>strSpc<~"'" | str) + val str: Regex = "[a-zA-Z0-9_]+".r + val strSpc: Regex = "[a-zA-Z0-9_ ]+".r + val realStr: Parser[String] = "\"" ~> strSpc <~ "\"" | "'" ~> strSpc <~ "'" | str - def expr: Parser[Seq[String]] = { + def expr: Parser[Seq[String]] = "[" ~ opt(realStr ~ rep("," ~ realStr)) ~ "]" ^^ { - case _ ~ Some(firstRole ~ list) ~ _ ⇒ list.foldLeft(Seq(firstRole)) { - case (queue, _ ~ role) ⇒ role +: queue - } - case _ ~ _ ⇒ Seq.empty[String] + case _ ~ Some(firstRole ~ list) ~ _ => + list.foldLeft(Seq(firstRole)) { + case (queue, _ ~ role) => role +: queue + } + case _ ~ _ => Seq.empty[String] } | opt(realStr) ^^ { - case Some(role) ⇒ Seq(role) - case None ⇒ Seq.empty[String] + case Some(role) => Seq(role) + case None => Seq.empty[String] } - } } - override def getUserFields(jsValue: JsValue, authHeader: Option[(String, String)]): Future[Fields] = { - + override def getUserFields(jsValue: JsValue, authHeader: Option[(String, String)]): Future[Fields] = groupsUrl match { - case Some(groupsEndpointUrl) ⇒ { - logger.debug(s"Retreiving groups from ${groupsEndpointUrl}") - val apiCall = authHeader.fold(ws.url(groupsEndpointUrl))(headers ⇒ ws.url(groupsEndpointUrl).addHttpHeaders(headers)) - apiCall.get.flatMap { r ⇒ extractGroupsThenBuildUserFields(jsValue, r.json) } - } - case None ⇒ { + case Some(groupsEndpointUrl) => + logger.debug(s"Retreiving groups from $groupsEndpointUrl") + val apiCall = authHeader.fold(ws.url(groupsEndpointUrl))(headers => ws.url(groupsEndpointUrl).addHttpHeaders(headers)) + apiCall.get.flatMap { r => + extractGroupsThenBuildUserFields(jsValue, r.json) + } + case None => logger.debug(s"Extracting groups from user info") extractGroupsThenBuildUserFields(jsValue, jsValue) - } } - } - private def extractGroupsThenBuildUserFields(jsValue: JsValue, groupsContainer: JsValue): Future[Fields] = { - (groupsContainer \ groupsAttrName) match { + private def extractGroupsThenBuildUserFields(jsValue: JsValue, groupsContainer: JsValue): Future[Fields] = + groupsContainer \ groupsAttrName match { // Groups received as valid JSON array - case JsDefined(JsArray(groupsList)) ⇒ mapGroupsAndBuildUserFields(jsValue, groupsList.map(_.as[String]).toList) + case JsDefined(JsArray(groupsList)) => mapGroupsAndBuildUserFields(jsValue, groupsList.map(_.as[String]).toList) // Groups list received as string (invalid JSON, for example: "ROLE" or "['Role 1', ROLE2, 'Role_3']") - case JsDefined(JsString(groupsStr)) ⇒ { + case JsDefined(JsString(groupsStr)) => val parser = new RoleListParser parser.parseAll(parser.expr, groupsStr) match { - case parser.Success(result, _) ⇒ mapGroupsAndBuildUserFields(jsValue, result) - case err: parser.NoSuccess ⇒ Future.failed(AuthenticationError(s"User info fails: can't parse groups list (${err.msg})")) + case parser.Success(result, _) => mapGroupsAndBuildUserFields(jsValue, result) + case err: parser.NoSuccess => Future.failed(AuthenticationError(s"User info fails: can't parse groups list (${err.msg})")) } - } // Invalid group list - case JsDefined(error) ⇒ - Future.failed(AuthenticationError(s"User info fails: invalid groups list received in user info ('${error}' of type ${error.getClass})")) + case JsDefined(error) => + Future.failed(AuthenticationError(s"User info fails: invalid groups list received in user info ('$error' of type ${error.getClass})")) // Groups field is undefined - case _: JsUndefined ⇒ - Future.failed(AuthenticationError(s"User info fails: groups attribute ${groupsAttrName} doesn't exist in user info")) + case _: JsUndefined => + Future.failed(AuthenticationError(s"User info fails: groups attribute $groupsAttrName doesn't exist in user info")) } - } private def mapGroupsAndBuildUserFields(jsValue: JsValue, jsonGroups: Seq[String]): Future[Fields] = { val mappedRoles = jsonGroups.flatMap(mappings.get).flatten.toSet @@ -106,6 +102,7 @@ class GroupUserMapper( if (roles.isEmpty) { Future.failed(AuthorizationError(s"No matched roles for user")) + } else { logger.debug(s"Computed roles: ${roles.mkString(", ")}") @@ -119,7 +116,8 @@ class GroupUserMapper( } yield Fields(Json.obj("login" -> login.toLowerCase, "name" -> name, "roles" -> roles, "organization" -> organization)) fields match { case JsSuccess(f, _) => Future.successful(f) - case JsError(errors) ⇒ Future.failed(AuthenticationError(s"User info fails: ${errors.map(_._2).map(_.map(_.messages.mkString(", ")).mkString("; ")).mkString}")) + case JsError(errors) => + Future.failed(AuthenticationError(s"User info fails: ${errors.map(_._2).map(_.map(_.messages.mkString(", ")).mkString("; ")).mkString}")) } } } From 026f2a76997ead3718f6dd62fcc3a6140c793ccd Mon Sep 17 00:00:00 2001 From: o101010 <23003062+o101010@users.noreply.github.com> Date: Tue, 20 Apr 2021 18:25:50 +0200 Subject: [PATCH 8/9] Update cortex.service add /etc/default/cortex to cortex.service script, like thehive4 have --- package/cortex.service | 1 + 1 file changed, 1 insertion(+) diff --git a/package/cortex.service b/package/cortex.service index f67192759..0e34c53ab 100644 --- a/package/cortex.service +++ b/package/cortex.service @@ -5,6 +5,7 @@ Wants=network-online.target After=network-online.target [Service] +EnvironmentFile=-/etc/default/cortex WorkingDirectory=/opt/cortex User=cortex From 0b49ab17ca22082918329bd7b7c81ca68a7e0370 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 21 Sep 2023 11:00:14 +0200 Subject: [PATCH 9/9] Release 3.1.8 --- .drone.yml | 182 ----------------------------------------------- CHANGELOG.md | 11 +++ version.sbt | 2 +- www/package.json | 2 +- 4 files changed, 13 insertions(+), 184 deletions(-) delete mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 10c087381..000000000 --- a/.drone.yml +++ /dev/null @@ -1,182 +0,0 @@ ---- -kind: pipeline -name: default -type: docker - -steps: - # Restore cache of downloaded dependencies - - name: restore-cache - image: drillster/drone-volume-cache - settings: - restore: true - mount: - - .sbt - - .ivy2 - - ui/node_modules - - ui/bower_components - volumes: [{name: cache, path: /cache}] - - # Run project tests - - name: run-tests - image: thehiveproject/drone-scala-node - commands: - - | - . ~/.nvm/nvm.sh - nvm install 12.18 - npm install -g bower - sbt -Duser.home=$PWD test:compile test Universal/packageBin - - # Build packages - - name: build-packages - image: thehiveproject/drone-scala-node - settings: - pgp_key: {from_secret: pgp_key} - commands: - - | - V=$(sbt -no-colors --error "print cortex/version" | tail -1) - if ( echo $V | grep -qi snapshot) - then - exit 1 - fi - . ~/.nvm/nvm.sh - nvm install 12.18 - npm install -g bower - [ -n "$PLUGIN_PGP_KEY" ] && gpg --batch --import - <<< $PLUGIN_PGP_KEY - sbt -Duser.home=$PWD Docker/stage Debian/packageBin Rpm/packageBin Universal/packageBin cortexWithDeps/Docker/stage - if ( echo $V | grep -qi rc ) - then - echo $( echo $V | sed -re 's/([0-9]+.[0-9]+.[0-9]+)-RC([0-9]+)-([0-9]+)/\1-RC\2,\1-RC\2-\3/' ) > .tags - else - echo $( echo $V | sed -re 's/([0-9]+).([0-9]+).([0-9]+)-([0-9]+)/\1,\1.\2,\1.\2.\3,\1.\2.\3-\4,latest/' ) > .tags - fi - echo $V > cortex-version.txt - mv target/rpm/RPMS/noarch/cortex*.rpm target/ - mv target/universal/cortex*.zip target/ - when: - event: [tag] - - # Save external libraries in cache - - name: save-cache - image: drillster/drone-volume-cache - settings: - rebuild: true - backend: "filesystem" - mount: - - .sbt - - .ivy2 - - .cache - - ui/node_modules - - ui/bower_components - volumes: [{name: cache, path: /cache}] - - # Send packages using scp - - name: send packages - image: appleboy/drone-scp - settings: - host: {from_secret: package_host} - username: {from_secret: package_user} - key: {from_secret: package_key} - target: {from_secret: incoming_path} - source: - - target/cortex*.deb - - target/cortex*.rpm - - target/cortex*.zip - strip_components: 1 - when: - event: [tag] - - # Publish packages - - name: publish packages - image: appleboy/drone-ssh - settings: - host: {from_secret: package_host} - user: {from_secret: package_user} - key: {from_secret: package_key} - publish_script: {from_secret: publish_script} - commands: - - PLUGIN_SCRIPT="bash $PLUGIN_PUBLISH_SCRIPT cortex $(cat cortex-version.txt)" /bin/drone-ssh - when: - event: [tag] - - # Publish docker image on Docker Hub - - name: docker - image: plugins/docker - settings: - context: target/docker/stage - dockerfile: target/docker/stage/Dockerfile - repo: thehiveproject/cortex - username: {from_secret: docker_username} - password: {from_secret: docker_password} - when: - event: [tag] - - # Publish docker image on Harbor - - name: harbor - image: plugins/docker - settings: - context: target/docker/stage - dockerfile: target/docker/stage/Dockerfile - registry: {from_secret: harbor_registry} - repo: {from_secret: harbor_repo} - username: {from_secret: harbor_username} - password: {from_secret: harbor_password} - when: - event: [tag] - - - name: update docker tags - image: thehiveproject/drone-scala-node - commands: - - sed -i -e 's/,/-withdeps,/g; s/$/-withdeps/' .tags - when: - event: [tag] - - # Publish docker image on Docker Hub - - name: docker fat - image: plugins/docker - settings: - context: target/docker-withdeps/target/docker/stage - dockerfile: target/docker-withdeps/target/docker/stage/Dockerfile - repo: thehiveproject/cortex - username: {from_secret: docker_username} - password: {from_secret: docker_password} - when: - event: [tag] - - # Publish docker image on Harbor - - name: harbor fat - image: plugins/docker - settings: - context: target/docker-withdeps/target/docker/stage - dockerfile: target/docker-withdeps/target/docker/stage/Dockerfile - registry: {from_secret: harbor_registry} - repo: {from_secret: harbor_repo} - username: {from_secret: harbor_username} - password: {from_secret: harbor_password} - when: - event: [tag] - - - name: send message - image: thehiveproject/drone_keybase - settings: - username: {from_secret: keybase_username} - paperkey: {from_secret: keybase_paperkey} - channel: {from_secret: keybase_channel} - commands: - - | - keybase oneshot -u "$PLUGIN_USERNAME" --paperkey "$PLUGIN_PAPERKEY" - URL="$DRONE_SYSTEM_PROTO://$DRONE_SYSTEM_HOST/$DRONE_REPO/$DRONE_BUILD_NUMBER" - if [ $DRONE_BUILD_STATUS = "success" ] - then - keybase chat send "$PLUGIN_CHANNEL" ":white_check_mark: $DRONE_REPO: build succeeded $URL" - else - keybase chat send "$PLUGIN_CHANNEL" ":x: $DRONE_REPO: build failed $URL" - fi - when: - status: - - success - - failure - -volumes: - - name: cache - host: - path: /opt/drone/cache diff --git a/CHANGELOG.md b/CHANGELOG.md index a787de77b..263426f26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Change Log +## [3.1.8](https://github.com/TheHive-Project/Cortex/milestone/35) (2023-09-21) + +**Pull requests:** + +- Fixed user parameter name in application config sample [\#315](https://github.com/TheHive-Project/Cortex/pull/315) +- Update cortex.service [\#361](https://github.com/TheHive-Project/Cortex/pull/361) +- GroupUserMapper.scala: Backport fix from TheHive for group mapper functionality [\#438](https://github.com/TheHive-Project/Cortex/pull/438) +- [CTX-16] fix: don't use Elasticsearch scroll to find user by its API key [\#447](https://github.com/TheHive-Project/Cortex/pull/447) +- [CTX-17] fix: Update version of JFFI [\#448](https://github.com/TheHive-Project/Cortex/pull/448) +- Update deps [\#449](https://github.com/TheHive-Project/Cortex/pull/449) + ## [3.1.7](https://github.com/TheHive-Project/Cortex/milestone/34) (2022-10-07) **Implemented enhancements:** diff --git a/version.sbt b/version.sbt index 72cb9276a..6a635e516 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "3.1.7-1" +ThisBuild / version := "3.1.8-1" diff --git a/www/package.json b/www/package.json index c5f90d233..34da96b82 100755 --- a/www/package.json +++ b/www/package.json @@ -1,6 +1,6 @@ { "name": "cortex", - "version": "3.1.7", + "version": "3.1.8", "description": "A powerfull observable analysis engine", "license": "AGPL-3.0-or-later", "homepage": "https://github.com/TheHive-Project/Cortex",