From 8312491c245d85fb77741cca74daccf2fc49faff Mon Sep 17 00:00:00 2001 From: Shachar Mossek Date: Sat, 22 Aug 2020 22:31:05 +0300 Subject: [PATCH] Version 0.0.13 Added ManicTime Web/Desktop field and set applicationName accordingly Added ManicTime support for user switching Fixed ManicTime filter in query instead of after (performance boost) Update system.yml for manictime only --- metricbeat/build.sh | 4 +- metricbeat/metricbeat.yml | 112 ++++++++-- .../module/system/manictime/manictime.go | 206 ++++++++++++------ metricbeat/modules.d/system.yml | 51 ++--- 4 files changed, 262 insertions(+), 111 deletions(-) diff --git a/metricbeat/build.sh b/metricbeat/build.sh index e398680fb96..e5562fe2389 100644 --- a/metricbeat/build.sh +++ b/metricbeat/build.sh @@ -2,5 +2,5 @@ make collect make -go build -o metricbeat_0.0.11 main.go -GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc go build -o metricbeat_0.0.11.exe main.go +go build -o metricbeat_0.0.13 main.go +GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc go build -o metricbeat_0.0.13.exe main.go diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 6ffee0fff80..8f37aa96ed3 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -7,6 +7,34 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/metricbeat/index.html +#================================= Paths ====================================== + +# The home path for the Metricbeat installation. This is the default base path +# for all other path settings and for miscellaneous files that come with the +# distribution (for example, the sample dashboards). +# If not set by a CLI flag or in the configuration file, the default for the +# home path is the location of the binary. +#path.home: /metricbeat-7.4.1-linux-x86_64 + +# The configuration path for the Metricbeat installation. This is the default +# base path for configuration files, including the main YAML configuration file +# and the Elasticsearch template file. If not set by a CLI flag or in the +# configuration file, the default for the configuration path is the home path. +#path.config: ${path.home} + +# The data path for the Metricbeat installation. This is the default base path +# for all the files in which Metricbeat needs to store its data. If not set by a +# CLI flag or in the configuration file, the default for the data path is a data +# subdirectory inside the home path. +#path.data: ${path.home}/data +#path.data: ${path.home}/data + +# The logs path for a Metricbeat installation. This is the default location for +# the Beat's log files. If not set by a CLI flag or in the configuration file, +# the default for the logs path is a logs subdirectory inside the home path. +#path.logs: ${path.home}/logs +path.logs: ${path.home}/logs + #========================== Modules configuration ============================ metricbeat.config.modules: @@ -14,10 +42,10 @@ metricbeat.config.modules: path: ${path.config}/modules.d/*.yml # Set to true to enable config reloading - reload.enabled: false + reload.enabled: true # Period on which files under path should be checked for changes - #reload.period: 10s + reload.period: 60s #==================== Elasticsearch template setting ========================== @@ -38,8 +66,12 @@ setup.template.settings: # Optional fields that you can specify to add additional information to the # output. +fields_under_root: true #fields: -# env: staging +# city: ${city} +# neighborhood: ${neighborhood} +# branch: ${branch} +# title: ${title} #============================== Dashboards ===================================== @@ -88,23 +120,68 @@ setup.kibana: # Configure what output to use when sending the data collected by the beat. +#------------------------------- File output ----------------------------------- +#output.file: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Path to the directory where to save the generated files. The option is + # mandatory. + #path: ${path.home}/output + + # Name of the generated files. The default is `metricbeat` and it generates + # files: `metricbeat`, `metricbeat.1`, `metricbeat.2`, etc. + #filename: metricbeat.out.json.log + + # Maximum size in kilobytes of each file. When this size is reached, and on + # every Metricbeat restart, the files are rotated. The default value is 10240 + # kB. + #rotate_every_kb: 10000 + + # Maximum number of files under path. When this number of files is reached, + # the oldest file is deleted and the rest are shifted from last to first. The + # default is 7 files. + #number_of_files: 9 + + # Permissions to use for file creation. The default is 0600. + #permissions: 0666 + +#----------------------------- Console output --------------------------------- +#output.console: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + #-------------------------- Elasticsearch output ------------------------------ -output.elasticsearch: +#output.elasticsearch: # Array of hosts to connect to. - hosts: ["localhost:9200"] + #hosts: ["localhost:9200"] - # Protocol - either `http` (default) or `https`. + # Optional protocol and basic auth credentials. #protocol: "https" - - # Authentication credentials - either API key or username/password. - #api_key: "id:api_key" #username: "elastic" #password: "changeme" #----------------------------- Logstash output -------------------------------- -#output.logstash: +output.logstash: # The Logstash hosts - #hosts: ["localhost:5044"] + # hosts: ["logstash01.westus2.cloudapp.azure.com:5044"] + hosts: ["localhost:5000"] # Optional SSL. By default is off. # List of root certificates for HTTPS server verifications @@ -123,8 +200,6 @@ output.elasticsearch: processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - - add_docker_metadata: ~ - - add_kubernetes_metadata: ~ #================================ Logging ===================================== @@ -163,3 +238,14 @@ processors: # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + +#chrome extension metricset settings +chrome_extension_metricset_settings: + port: 8006 + showInMin: true + +manictime_metricset_settings: + path: C:\Users\%USER%\AppData\Local\Finkit\ManicTime\ManicTimeReports.db + defaultUser: default + webKeys: + - chrome.exe;google chrome diff --git a/metricbeat/module/system/manictime/manictime.go b/metricbeat/module/system/manictime/manictime.go index 349b0f5b282..867a227243a 100644 --- a/metricbeat/module/system/manictime/manictime.go +++ b/metricbeat/module/system/manictime/manictime.go @@ -32,13 +32,17 @@ func init() { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - database *sql.DB - userName string + database *sql.DB + databasePathTemplate string + defaultUserName string + lastUserName string + webKeys []string } type ManicTimeConfig struct { - Path string `yaml:"path"` - User string `yaml:"user"` + Path string `yaml:"path"` + DefaultUser string `yaml:"defaultUser"` + WebKeys []string `yaml:"webKeys"` } type Config struct { @@ -68,6 +72,7 @@ type ValidActivity struct { id string durationMin float64 durationSec float64 + applicationType string applicationName string } @@ -86,19 +91,25 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // read config var cfg Config - readFile(&cfg) + readConfigFile(&cfg) + + currentUserName := getCurrentUserName(cfg.Settings.DefaultUser) + databasePath := getUserDatabasePath(cfg.Settings.Path, currentUserName) // connect to db - database, err := sql.Open("sqlite3", cfg.Settings.Path) + database, err := sql.Open("sqlite3", databasePath) if err != nil { - fmt.Println("could not open db file of manicTime") + database = nil + fmt.Println("could not open db file of manicTime", databasePath) } // get current username - userName := getUserName(cfg) return &MetricSet{ - BaseMetricSet: base, - database: database, - userName: userName, + BaseMetricSet: base, + databasePathTemplate: cfg.Settings.Path, + database: database, + defaultUserName: cfg.Settings.DefaultUser, + lastUserName: currentUserName, + webKeys: cfg.Settings.WebKeys, }, nil } @@ -106,13 +117,46 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { + currentUserName := getCurrentUserName(m.defaultUserName) + if currentUserName != m.lastUserName { + // Get last user name data + generateData(m, m.lastUserName, report) + + // Set current user data + databasePath := getUserDatabasePath(m.databasePathTemplate, currentUserName) + database, err := sql.Open("sqlite3", databasePath) + if err != nil { + fmt.Println("Switching user but failed to load database", databasePath) + database = nil + } + m.database = database + m.lastUserName = currentUserName + } else if m.database == nil { + // Try to open database again + databasePath := getUserDatabasePath(m.databasePathTemplate, currentUserName) + database, err := sql.Open("sqlite3", databasePath) + if err == nil { + fmt.Print("Successful recovery of database", databasePath) + m.database = database + } + } + + // Get current user name data + generateData(m, currentUserName, report) + return nil +} + +func generateData(m *MetricSet, username string, report mb.ReporterV2) { + if m.database == nil { + return + } lastSync := getLastSyncTime(m.database) // parse string(date) to time parsedLastSync, _ := time.Parse(time.RFC3339, lastSync) // get all data - newData := getManicTimeNewData(m.database, parsedLastSync) + newData := getManicTimeNewData(m.database, m.webKeys, parsedLastSync) for _, activity := range newData { rootFields := common.MapStr{ @@ -126,6 +170,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { "siteName": activity.siteName, "durationMin": activity.durationMin, "durationSec": activity.durationSec, + "applicationType": activity.applicationType, "applicationName": activity.applicationName, "id": activity.id, } @@ -135,20 +180,18 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { }, RootFields: common.MapStr{ "user": common.MapStr{ - "name": m.userName, + "name": username, }, }, }) } - // update lastsync (memory + table) + // update lastsync (memory + table) updateLastSync(m.database) - - return nil } // read config file -func readFile(cfg *Config) { +func readConfigFile(cfg *Config) { ex, err := os.Executable() if err != nil { panic(err) @@ -168,6 +211,13 @@ func readFile(cfg *Config) { } +func getUserDatabasePath(pathTemplate string, username string) string { + userParts := strings.Split(username, "\\") + username = userParts[len(userParts)-1] + databasePath := strings.Replace(pathTemplate, "%USER%", username, 1) + return databasePath +} + func getLastSyncTime(database *sql.DB) string { _, tableErr := database.Query(`SELECT lastSync FROM Sync WHERE id=1`) @@ -195,72 +245,86 @@ func getLastSyncTime(database *sql.DB) string { return lastSync } -func getManicTimeNewData(database *sql.DB, lastTimeSync time.Time) []ValidActivity { +func isContains(slice []string, item string) bool { + for _, x := range slice { + if x == item { + return true + } + } + return false +} + +var queryTemplate = `SELECT a.Name as title, a.StartUtcTime as startTime, a.EndUtcTime as endTime, b.Name as url, c.Key as appKey, c.Name as appName, d.Key as siteKey, d.Name as siteName +FROM "Ar_Activity" as a +LEFT JOIN "Ar_Activity" as b +ON a.ActivityId = b.RelatedActivityId +LEFT JOIN "Ar_CommonGroup" as c +ON a.CommonGroupId = c.CommonId +LEFT JOIN "Ar_CommonGroup" as d +ON b.CommonGroupId = d.CommonId +WHERE a.RelatedActivityId is NULL AND a.EndUtcTime >= '%v'` + +func getManicTimeNewData(database *sql.DB, webKeys []string, lastTimeSync time.Time) []ValidActivity { var newData []ValidActivity - rows, _ := database.Query(`SELECT a.Name as title, a.StartUtcTime as startTime, a.EndUtcTime as endTime, b.Name as url, c.Key as appKey, c.Name as appName, d.Key as siteKey, d.Name as siteName - FROM "Ar_Activity" as a - LEFT JOIN "Ar_Activity" as b - ON a.ActivityId = b.RelatedActivityId - LEFT JOIN "Ar_CommonGroup" as c - ON a.CommonGroupId = c.CommonId - LEFT JOIN "Ar_CommonGroup" as d - ON b.CommonGroupId = d.CommonId - WHERE a.RelatedActivityId is NULL`) + rows, _ := database.Query(fmt.Sprintf(queryTemplate, lastTimeSync)) for rows.Next() { p := Activity{} err := rows.Scan(&p.title, &p.startTime, &p.endTime, &p.url, &p.appKey, &p.appName, &p.siteKey, &p.siteName) if err != nil { fmt.Println("error scanning rows in manictime metricset", err) + continue } - if p.title == "Active" { continue } - parsedEndTime, _ := time.Parse(time.RFC3339, p.endTime) - if parsedEndTime.After(lastTimeSync) { - newActivity := ValidActivity{} - newActivity.title = p.title - newActivity.startTime = getMaxStartTime(p.startTime, lastTimeSync) - newActivity.endTime = p.endTime - if p.url.Valid { - newActivity.url = p.url.String - } else { - newActivity.url = "" - } - if p.appKey.Valid { - newActivity.appKey = p.appKey.String - } else { - newActivity.appKey = "" - } - if p.appName.Valid { - newActivity.appName = p.appName.String - } else { - newActivity.appName = "" - } - if p.siteKey.Valid { - newActivity.siteKey = p.siteKey.String - } else { - newActivity.siteKey = "" - } - if p.siteName.Valid { - newActivity.siteName = p.siteName.String - newActivity.applicationName = p.siteName.String - } else { - newActivity.siteName = "" - newActivity.applicationName = newActivity.appName - } - startTime, _ := time.Parse(time.RFC3339, newActivity.startTime) - endTime, _ := time.Parse(time.RFC3339, newActivity.endTime) - newActivity.durationMin = toFixed(endTime.Sub(startTime).Minutes(), 3) - newActivity.durationSec = endTime.Sub(startTime).Seconds() - hostname, _ := os.Hostname() - id := newActivity.appName + "_" + newActivity.startTime + "_" + hostname - newActivity.id = strings.ReplaceAll(id, " ", "_") - newData = append(newData, newActivity) + newActivity := ValidActivity{} + newActivity.title = p.title + newActivity.startTime = getMaxStartTime(p.startTime, lastTimeSync) + newActivity.endTime = p.endTime + if p.url.Valid { + newActivity.url = p.url.String + } else { + newActivity.url = "" + } + if p.appKey.Valid { + newActivity.appKey = p.appKey.String + } else { + newActivity.appKey = "" + } + if p.appName.Valid { + newActivity.appName = p.appName.String + } else { + newActivity.appName = "" + } + if p.siteKey.Valid { + newActivity.siteKey = p.siteKey.String + } else { + newActivity.siteKey = "" + } + if p.siteName.Valid { + newActivity.siteName = p.siteName.String + } else { + newActivity.siteName = "" + } + if p.appKey.Valid && isContains(webKeys, p.appKey.String) { + newActivity.applicationType = "Web" + newActivity.applicationName = p.siteName.String + } else { + newActivity.applicationType = "Desktop" + newActivity.applicationName = newActivity.appName } + startTime, _ := time.Parse(time.RFC3339, newActivity.startTime) + endTime, _ := time.Parse(time.RFC3339, newActivity.endTime) + timeDiff := endTime.Sub(startTime) + newActivity.durationMin = toFixed(timeDiff.Minutes(), 3) + newActivity.durationSec = toFixed(timeDiff.Seconds(), 3) + hostname, _ := os.Hostname() + id := newActivity.appName + "_" + newActivity.startTime + "_" + hostname + newActivity.id = strings.ReplaceAll(id, " ", "_") + newData = append(newData, newActivity) } return newData } @@ -279,7 +343,7 @@ func getMaxStartTime(startTime string, lastTimeSync time.Time) string { return parsedStartTime.Format(time.RFC3339) } -func getUserName(cfg Config) string { +func getCurrentUserName(defaultUser string) string { var username string var query []Win32_ComputerSystem @@ -290,7 +354,7 @@ func getUserName(cfg Config) string { if query[0].UserName != "" { username = query[0].UserName } else { - username = cfg.Settings.User + username = defaultUser } return username diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index 2388168b80b..882d5352438 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -1,38 +1,39 @@ -# Module: system -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/7.6/metricbeat-module-system.html - +# Module: system +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/7.6/metricbeat-module-system.html + - module: system period: 10s metricsets: - - cpu - - load - - memory - - network - - process - - process_summary - - socket_summary + # - cpu + # - load + # - memory + # - network + # - process + # - process_summary + # - socket_summary #- entropy #- core #- diskio #- socket #- services - process.include_top_n: - by_cpu: 5 # include top 5 processes by CPU - by_memory: 5 # include top 5 processes by memory + - manictime + # process.include_top_n: + # by_cpu: 5 # include top 5 processes by CPU + # by_memory: 5 # include top 5 processes by memory -- module: system - period: 1m - metricsets: - - filesystem - - fsstat - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' +# - module: system +# period: 1m +# metricsets: +# - filesystem +# - fsstat +# processors: +# - drop_event.when.regexp: +# system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' -- module: system - period: 15m - metricsets: - - uptime +# - module: system +# period: 15m +# metricsets: +# - uptime #- module: system # period: 5m