-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Update to Angular 13 and Node 16 * Adding a report to show the usage of rules as imported from web analytics --------- Co-authored-by: mkr <[email protected]>
- Loading branch information
Showing
25 changed files
with
452 additions
and
50 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31,3 +31,6 @@ hs_err_pid* | |
|
|
||
| # SBT related stuff | ||
| .bsp | ||
|
|
||
| # Ignore the Angular Cache | ||
| frontend/.angular/cache | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| package models.reports | ||
|
|
||
| import models.input.{SearchInput, SearchInputId, SearchInputWithRules} | ||
| import play.api.Logging | ||
| import play.api.libs.json.{Json, OFormat} | ||
| import services.RulesUsage | ||
|
|
||
| case class RulesUsageReportEntry( | ||
| // the search input ID from the report | ||
| searchInputId: String, | ||
| // the stored search input term if it could be found in the DB | ||
| searchInputTerm: Option[String], | ||
| // the user keywords/query that the search input was triggered on | ||
| keywords: String, | ||
| // the frequency that rule was triggered for the particular keywords | ||
| frequency: Int | ||
| ) | ||
|
|
||
| case class RulesUsageReport(items: Seq[RulesUsageReportEntry]) | ||
|
|
||
| object RulesUsageReport extends Logging { | ||
|
|
||
| implicit val jsonFormatRulesUsageReportEntry: OFormat[RulesUsageReportEntry] = Json.format[RulesUsageReportEntry] | ||
| implicit val jsonFormatRulesUsageReport: OFormat[RulesUsageReport] = Json.format[RulesUsageReport] | ||
|
|
||
| def create(searchInputs: Seq[SearchInputWithRules], rulesUsageStatistics: Seq[RulesUsage]): RulesUsageReport = { | ||
| // perform a "full outer join" of the rules usage with the existing search inputs | ||
| val searchInputsById = searchInputs.map(searchInput => searchInput.id.id -> searchInput).toMap | ||
| val searchInputIdsFromAnalytics = rulesUsageStatistics.map(_.inputId.id).toSet | ||
| val searchInputIdsNotFound = searchInputIdsFromAnalytics -- searchInputsById.keySet | ||
| val searchInputIdsNotUsed = searchInputsById.keySet -- searchInputIdsFromAnalytics | ||
| logger.info(s"Creating report from ${searchInputIdsFromAnalytics.size} used search inputs" + | ||
| s" and ${searchInputsById.size} search inputs currently configured" + | ||
| s" with ${searchInputIdsNotFound.size} search inputs not found" + | ||
| s" and ${searchInputIdsNotUsed.size} search inputs not used") | ||
|
|
||
| val reportEntriesUsedSearchInputs = rulesUsageStatistics.map { rulesUsage: RulesUsage => | ||
| RulesUsageReportEntry( | ||
| rulesUsage.inputId.id, | ||
| searchInputsById.get(rulesUsage.inputId.id).map(_.term), | ||
| rulesUsage.keywords, | ||
| rulesUsage.frequency | ||
| ) | ||
| } | ||
| val reportEntriesUnusedSearchInputs = searchInputIdsNotUsed.map { searchInputId => | ||
| RulesUsageReportEntry( | ||
| searchInputId, | ||
| searchInputsById.get(searchInputId).map(_.term), | ||
| "", | ||
| 0 | ||
| ) | ||
| } | ||
| RulesUsageReport(reportEntriesUsedSearchInputs ++ reportEntriesUnusedSearchInputs) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| package services | ||
|
|
||
| import com.google.cloud.storage.{BlobId, StorageOptions} | ||
|
|
||
| import java.io.{InputStreamReader, Reader} | ||
| import java.net.URI | ||
| import java.nio.channels.Channels | ||
| import java.nio.charset.StandardCharsets | ||
| import java.nio.file.{Files, Paths} | ||
|
|
||
| trait ReaderProvider { | ||
| def openReader(url: String): Reader | ||
| } | ||
|
|
||
| class FileSystemReaderProvider extends ReaderProvider { | ||
| override def openReader(url: String): Reader = { | ||
| val uri = if (url.startsWith("file://")) url else "file://" + url | ||
| val path = Paths.get(new URI(uri)) | ||
| Files.newBufferedReader(path) | ||
| } | ||
| } | ||
|
|
||
| class GcsReaderProvider extends ReaderProvider { | ||
|
|
||
| override def openReader(url: String): Reader = { | ||
| val storage = StorageOptions.getDefaultInstance.getService | ||
| val blob = storage.get(BlobId.fromGsUtilUri(url)) | ||
| val readChannel = blob.reader() | ||
| new InputStreamReader(Channels.newInputStream(readChannel), StandardCharsets.UTF_8) | ||
| } | ||
| } | ||
|
|
||
| class ReaderProviderDispatcher extends ReaderProvider { | ||
|
|
||
| private val fileSystemReaderProvider = new FileSystemReaderProvider() | ||
| private lazy val gcsReaderProvider: GcsReaderProvider = { | ||
| new GcsReaderProvider() | ||
| } | ||
|
|
||
| override def openReader(url: String): Reader = { | ||
| val readerProvider = url.toLowerCase.trim match { | ||
| case url if url.startsWith("gs://") => gcsReaderProvider | ||
| case url if url.startsWith("file://") => fileSystemReaderProvider | ||
| case url if Files.exists(Paths.get(url)) => fileSystemReaderProvider | ||
| case _ => throw new IllegalArgumentException(s"Unsupported URL scheme or file not found: ${url}") | ||
| } | ||
| readerProvider.openReader(url) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| package services | ||
|
|
||
| import models.input.SearchInputId | ||
| import org.apache.commons.csv.CSVFormat | ||
| import play.api.{Configuration, Logging} | ||
|
|
||
| import javax.inject.Inject | ||
| import scala.collection.JavaConverters.iterableAsScalaIterableConverter | ||
|
|
||
| case class RulesUsage(inputId: SearchInputId, | ||
| keywords: String, | ||
| frequency: Int) | ||
|
|
||
| class RulesUsageService @Inject()(configuration: Configuration, | ||
| readerProvider: ReaderProviderDispatcher) extends Logging { | ||
|
|
||
| private val CsvFormat = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).build() | ||
|
|
||
| def getRulesUsageStatistics: Option[Seq[RulesUsage]] = { | ||
| configuration.getOptional[String](RulesUsageService.ConfigKeyRuleUsageStatistics) | ||
| .map { location => | ||
| logger.info(s"Loading rule usage statistics from ${location}") | ||
| try { | ||
| val reader = readerProvider.openReader(location) | ||
| try { | ||
| CsvFormat.parse(reader).asScala.map { record => | ||
| RulesUsage( | ||
| SearchInputId(record.get("SMUI_GUID")), | ||
| record.get("USER_QUERY"), | ||
| record.get("FREQUENCY").toInt) | ||
| }.toSeq | ||
| } finally { | ||
| reader.close() | ||
| } | ||
| } catch { | ||
| case e: Exception => | ||
| logger.error("Could not load rule usage statistics", e) | ||
| Seq.empty | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } | ||
|
|
||
| object RulesUsageService { | ||
|
|
||
| val ConfigKeyRuleUsageStatistics = "smui.rule-usage-statistics.location" | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,7 @@ | ||
| import com.typesafe.sbt.GitBranchPrompt | ||
|
|
||
| name := "search-management-ui" | ||
| version := "4.0.11" | ||
| version := "4.1.0" | ||
| maintainer := "Contact productful.io <[email protected]>" | ||
|
|
||
| scalaVersion := "2.12.17" | ||
|
|
@@ -67,10 +67,12 @@ libraryDependencies ++= { | |
| "com.typesafe.play" %% "play-json" % "2.9.3", | ||
| "com.pauldijou" %% "jwt-play" % "4.1.0", | ||
| "com.fasterxml.jackson.module" %% "jackson-module-scala" % JacksonVersion, | ||
| "org.apache.commons" % "commons-csv" % "1.10.0", | ||
| "org.apache.shiro" % "shiro-core" % "1.12.0", | ||
| "org.pac4j" % "pac4j-http" % Pac4jVersion excludeAll (JacksonCoreExclusion, BcProv15Exclusion, SpringJclBridgeExclusion), | ||
| "org.pac4j" % "pac4j-saml" % Pac4jVersion excludeAll (JacksonCoreExclusion, BcProv15Exclusion, SpringJclBridgeExclusion), | ||
| "org.pac4j" %% "play-pac4j" % "11.1.0-PLAY2.8", | ||
| "com.google.cloud" % "google-cloud-storage" % "2.33.0", | ||
| "org.scalatestplus.play" %% "scalatestplus-play" % "3.1.0" % Test, | ||
| "org.mockito" % "mockito-all" % "1.10.19" % Test, | ||
| "com.pauldijou" %% "jwt-play" % "4.1.0", | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.