diff --git a/docs/Showback.md b/docs/Showback.md new file mode 100644 index 00000000..b016528f --- /dev/null +++ b/docs/Showback.md @@ -0,0 +1,8 @@ +# Notes about configuring VM Showback + +## BILLING_PERIOD + +Each and every VM should have BILLING_PERIOD defined in template. +Possible values are: + - PAYG - VM will be billed per second + - PRE_<% n_days %> - Pre-Paid per N days, e.g. `PRE_30`, then full Capacity and Disk cost will be charged on deploy, and then every 30 days. \ No newline at end of file diff --git a/hooks/insert_zero_traffic_record.rb b/hooks/insert_zero_traffic_record.rb new file mode 100644 index 00000000..3556a282 --- /dev/null +++ b/hooks/insert_zero_traffic_record.rb @@ -0,0 +1,43 @@ +#!/usr/bin/env ruby +# -------------------------------------------------------------------------- # +# Copyright 2020, IONe Cloud Project, Support.by # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# -------------------------------------------------------------------------- # + +require 'base64' +require 'nokogiri' + +xml = Nokogiri::XML(Base64::decode64(ARGV.first)) +unless xml.xpath("/CALL_INFO/RESULT").text.to_i == 1 then + puts "VM wasn't allocated, skipping" + exit 0 +end + +vmid = xml.xpath('//EXTRA/VM') + +require 'yaml' +require 'json' +require 'sequel' + +$ione_conf = YAML.load_file("/etc/one/ione.conf") # IONe configuration constants + +require $ione_conf['DB']['adapter'] +$db = Sequel.connect({ + adapter: $ione_conf['DB']['adapter'].to_sym, + user: $ione_conf['DB']['user'], password: $ione_conf['DB']['pass'], + database: $ione_conf['DB']['database'], host: $ione_conf['DB']['host'] }) + +$db[:traffic_records].insert( + vm: vmid, rx: "0", tx: "0", rx_last: "0", tx_last: "0", stime: Time.now.to_i +) \ No newline at end of file diff --git a/ione_server.rb b/ione_server.rb index 7661f6e0..1a0a85f3 100644 --- a/ione_server.rb +++ b/ione_server.rb @@ -115,6 +115,8 @@ class Settings < Sequel::Model(:settings); end puts 'Including on_helper funcs' require "#{ROOT}/service/on_helper.rb" include ONeHelper +puts 'Including showback' +require "#{ROOT}/service/showback.rb" puts 'Including Deferable module' require "#{ROOT}/service/defer.rb" diff --git a/models/SettingsDriver.rb b/models/SettingsDriver.rb index c6ad2a57..35caf48a 100644 --- a/models/SettingsDriver.rb +++ b/models/SettingsDriver.rb @@ -9,15 +9,17 @@ required = [ ['ALERT', "0.0", "Balance, when user will be alerted", 0, "num"], - ['CAPACITY_COST', "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}", "VM Capacity resources costs", 1, "object"], + ['CAPACITY_COST', "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}", "VM Capacity resources costs per sec", 1, "object"], ['DISK_TYPES', "HDD,SSD,NVMe", "Comma-separated list of existing disk types", 1, "list"], - ['DISK_COSTS', "{\"disk_type\":\"price\"}", "Costs of different disk types", 1, "object"], + ['DISK_COSTS', "{\"disk_type\":\"price\"}", "Costs of different disk types GB/sec", 1, "object"], ['IAAS_GROUP_ID', 'iaas_group_id', "IaaS(VDC) Users group ID", 1, "num"], ['NODES_DEFAULT', "{\"hypervisor_name\":\"host_id\"}", "Default nodes for different hypervisors", 1, "object"], - ['PUBLIC_IP_COST', "0.0", "Public IP Address cost", 0, "num"], + ['PUBLIC_IP_COST', "0.0", "Public IP Address cost per sec", 0, "num"], ['PUBLIC_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Public Network Pool ID", 1, "object"], ['PRIVATE_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Private Network Pool ID", 1, "object"], - ['CURRENCY_MAIN', "€", "Currency", 0, "str"] + ['CURRENCY_MAIN', "€", "Currency", 0, "str"], + ['TRAFFIC_COST', "0.0", "Cost per 1 kByte traffic", 1, "num"], + ['SNAPSHOT_COST', "0.0", "Cost 1 Snapshot per sec", 1, "num"] ] required.each do | record | $db[:settings].insert(name: record[0], body: record[1], description: record[2], access_level: record[3], type: record[4]) diff --git a/rake/set_hooks.rake b/rake/set_hooks.rake index 84bfbb7e..c31c86cb 100644 --- a/rake/set_hooks.rake +++ b/rake/set_hooks.rake @@ -9,6 +9,13 @@ require 'opennebula' "COMMAND" => '/usr/lib/one/ione/hooks/set_price.rb', "ARGUMENTS" => '$API' }, + { + "NAME" => 'insert-zero-traffic-record', + "TYPE" => 'api', + "CALL" => 'one.vm.allocate', + "COMMAND" => '/usr/lib/one/ione/hooks/insert_zero_traffic_record.rb', + "ARGUMENTS" => '$API' + }, { "NAME" => 'pending', "ON" => "CUSTOM", diff --git a/rake/tests/Rakefile b/rake/tests/Rakefile new file mode 100644 index 00000000..91ffec91 --- /dev/null +++ b/rake/tests/Rakefile @@ -0,0 +1,17 @@ +task :before_test do + require '/usr/lib/one/ione/ione_server.rb' +end + +def passed + puts "--- " + "Passed".green +end +def fail msg + puts msg.red + exit +end +def warn msg + puts msg.yellow +end + +load "rake/tests/showback.rake" + diff --git a/rake/tests/showback.rake b/rake/tests/showback.rake new file mode 100644 index 00000000..8e1e0168 --- /dev/null +++ b/rake/tests/showback.rake @@ -0,0 +1,68 @@ +desc "Test Capacity Showback" +task :cap_showback_test => :before_test do + puts "\n####################\n# Testing capacity #\n####################" + + $db[:settings].where(name: "CAPACITY_COST").update(body: "{\"CPU_COST\":\"0.5\",\"MEMORY_COST\":\"0.5\"}") + $db[:settings].where(name: "DISK_COSTS").update(body: "{\"HDD\":\"0.0\"}") + + r = onblock(:vm, 3).calculate_showback 0, 1200 + if r[:TOTAL] == 600 then + passed + else + warn "#{0}, #{1200} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 1200 + if r[:TOTAL] == 300 then + passed + else + warn "#{300}, #{1200} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 500 + if r[:TOTAL] == 200 then + passed + else + warn "#{300}, #{500} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 1400 + if r[:TOTAL] == 500 then + passed + else + warn "#{300}, #{1400} => #{r[:TOTAL]}" + end +end + +desc "Test Disk Showback" +task :disk_showback_test => :before_test do + puts "\n################\n# Testing Disk #\n################" + + $db[:settings].where(name: "CAPACITY_COST").update(body: "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}") + $db[:settings].where(name: "DISK_COSTS").update(body: "{\"HDD\":\"1\"}") + + r = onblock(:vm, 3).calculate_showback 0, 1200 + if r[:TOTAL] == 1200 then + passed + else + warn "#{0}, #{1200} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 1200 + if r[:TOTAL] == 900 then + passed + else + warn "#{300}, #{1200} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 500 + if r[:TOTAL] == 200 then + passed + else + warn "#{300}, #{500} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 1400 + if r[:TOTAL] == 1100 then + passed + else + warn "#{300}, #{1400} => #{r[:TOTAL]}" + end +end + +desc "Showback Tests" +task :showback_test => [:cap_showback_test, :disk_showback_test] do; end \ No newline at end of file diff --git a/scripts/traffic-recorder/main.rb b/scripts/traffic-recorder/main.rb new file mode 100644 index 00000000..0c33425a --- /dev/null +++ b/scripts/traffic-recorder/main.rb @@ -0,0 +1,24 @@ +begin + LOG 'Traffic Recorder has been initialized', 'TrafficRecorder' + loop do + begin + vm_pool = VirtualMachinePool.new($client) + vm_pool.info_all + inserts_total = 0 + + vm_pool.each do | vm | + inserts = TrafficRecords.new(vm.id, true).sync vm + LOG "VM #{vm.id}: Inserted #{inserts} new traffic records", "TrafficRecorder" + inserts_total += inserts + end + + LOG "TrafficRecorder inserted totally #{inserts_total} new traffic records", 'TrafficRecorder' + sleep($ione_conf['TrafficRecorder']['check-period']) + rescue => e + LOG "TrafficRecorder Error, code: #{e.message}\nTrafficRecorder is down now", 'TrafficRecorder' + sleep(30) + end + end +rescue + LOG "TrafficRecorder fatal error, service is crashed", 'TrafficRecorderThread' +end \ No newline at end of file diff --git a/service/biller.rb b/service/biller.rb new file mode 100644 index 00000000..65d1480c --- /dev/null +++ b/service/biller.rb @@ -0,0 +1,26 @@ +class Biller + + def initialize vm + @vm = vm + end + + # Costs hash + def costs + SETTINGS_TABLE.as_hash(:name, :body).select {|key| key.include? 'COST' } + end + + # Check if this biller should be used in Billing + def check_biller + true + end + + def billing_period + @vm['//BILLING_PERIOD'] + end + + def bill bill:, state:, delta:, record: nil + 0 + end +end + +Dir["#{ROOT}/service/billers/*.rb"].each {|file| require file } diff --git a/service/billers/capacity.rb b/service/billers/capacity.rb new file mode 100644 index 00000000..c553730a --- /dev/null +++ b/service/billers/capacity.rb @@ -0,0 +1,24 @@ +class CapacityBiller < Biller + # Checking if Capacity costs are given, otherwise there is no point to calculate it + def check_biller + @costs = JSON.parse(costs['CAPACITY_COST']) + return false if @costs.nil? + + @cost = + @costs.values.inject(0) do | r, c | + r += c.to_f + rescue + r + end + return @cost > 0 + rescue + return false + end + + def bill bill:, state:, delta:, record: nil + if state[:state] == 'on' || billing_period != 'PAYG' then + bill[:capacity] = delta * @cost + end + bill + end +end \ No newline at end of file diff --git a/service/billers/disk.rb b/service/billers/disk.rb new file mode 100644 index 00000000..8f235f06 --- /dev/null +++ b/service/billers/disk.rb @@ -0,0 +1,26 @@ +class DiskBiller < Biller + # Checking if Capacity costs are given, otherwise there is no point to calculate it + def check_biller + @costs = JSON.parse(costs['DISK_COSTS']) + return false if @costs.nil? + + r = + @costs.values.inject(0) do | r, c | + r += c.to_f + rescue + r + end + return false if r <= 0 + + @cost = @costs[@vm['/VM/USER_TEMPLATE/DRIVE']].to_f + @size = vm.drives.inject(0) { | r, d | r += d['SIZE'].to_i } / 1000.0 + return @cost > 0 + rescue + return false + end + + def bill bill:, state:, delta:, record: nil + bill[:disk] = delta * @cost * @size + bill + end +end \ No newline at end of file diff --git a/service/billers/snapshots.rb b/service/billers/snapshots.rb new file mode 100644 index 00000000..24493cbb --- /dev/null +++ b/service/billers/snapshots.rb @@ -0,0 +1,16 @@ +class SnapshotsBiller < Biller + def check_biller + @cost = JSON.parse(costs['SNAPSHOT_COST']) + return false if @cost.nil? + + @cost = @cost.to_f + return @cost > 0 + rescue + return false + end + + def bill bill:, state:, delta:, record: nil + bill[:snapshots] = delta * @cost * state[:snaps] + bill + end +end \ No newline at end of file diff --git a/service/billers/traffic.rb b/service/billers/traffic.rb new file mode 100644 index 00000000..fff56062 --- /dev/null +++ b/service/billers/traffic.rb @@ -0,0 +1,25 @@ +class TrafficBiller < Biller + def check_biller + @cost = JSON.parse(costs['TRAFFIC_COST']) + return false if @cost.nil? + + @cost = @cost.to_f + @costs = { rx: @cost, tx: @cost } # Will Add support for differnt rx and tx prices + + return @costs.values.inject(0) do | r, c | + r += c.to_f + rescue + r + end > 0 + rescue + return false + end + + def bill bill:, state:, delta:, record: + if record.class == TrafficRecord then + bill[:rx] = state[:rx] / 1000.0 * @costs[:rx] + bill[:tx] = state[:tx] / 1000.0 * @costs[:tx] + end + bill + end +end \ No newline at end of file diff --git a/service/handlers/cache_handler.rb b/service/handlers/cache_handler.rb deleted file mode 100644 index 78b670ba..00000000 --- a/service/handlers/cache_handler.rb +++ /dev/null @@ -1,44 +0,0 @@ -# @!visibility private -class CacheStack - - include Enumerable - # Initializer of CacheStack object - def initialize(num = 5) - @size = num - @queue = Array.new - end - # Stack iterator - def each(&blk) - @queue.each(&blk) - end - # Stack pop method - def pop - @queue.pop - end - # Stack push method - def push(value) - @queue.shift if @queue.size >= @size - @queue.push(value) - end - # Stack as array - def to_a - @queue.to_a - end - # Alias for push - def <<(value) - push(value) - end - # Reads last object - def last - return @queue.last - end - # Gets object if include - def get_if_include(data) - return (self << @queue.delete(data)).last - end - # Returns string array of Stack - def to_s - return @queue - end - -end \ No newline at end of file diff --git a/service/log.rb b/service/log.rb index b75cecbe..7303f72d 100644 --- a/service/log.rb +++ b/service/log.rb @@ -31,11 +31,13 @@ def LOG(msg, method = "none", _time = true) destination = "#{LOG_ROOT}/debug.log" when "SnapController" destination = "#{LOG_ROOT}/snapshot.log" + when "TrafficRecorder" + destination = "#{LOG_ROOT}/traffic_recorder.log" else destination = "#{LOG_ROOT}/ione.log" end msg = msg.to_s - msg = "[ #{time()} ] " + msg if _time + msg = "[ #{Time.now.ctime} ] " + msg if _time msg += " [ #{method} ]" if method != 'none' && method != "" && method != nil File.open(destination, 'a'){ |log| log.write msg + "\n" } @@ -49,8 +51,9 @@ def LOG(msg, method = "none", _time = true) def LOG_COLOR(msg, method = caller_locations(1,1)[0].label.dup, color = 'red', font = 'bold') destination = "#{LOG_ROOT}/ione.log" destination = "#{LOG_ROOT}/snapshot.log" if method == "SnapController" + destination = "#{LOG_ROOT}/traffic_recorder.log" if method == "TrafficRecorder" msg = msg.to_s.send(color).send(font) - msg = "[ #{time()} ] " + msg + msg = "[ #{Time.now.ctime} ] " + msg method.slice!('block in '.dup) msg += " [ #{method} ]" if method != 'none' && method != "" && method != nil @@ -64,7 +67,7 @@ def LOG_COLOR(msg, method = caller_locations(1,1)[0].label.dup, color = 'red', f # Logging the message directly into LOG_LOCATION/debug.log def LOG_DEBUG(msg, method = 'DEBUG', _time = true) destination = "#{LOG_ROOT}/debug.log" - msg = "[ #{time()} ] #{msg}" + msg = "[ #{Time.now.ctime} ] #{msg}" File.open(destination, 'a'){ |log| log.write msg + "\n" } $log << "#{msg} | #{destination}" true @@ -83,11 +86,13 @@ def LOG_AUTO(msg, method = caller_locations(1,1)[0].label, _time = true) destination = "#{LOG_ROOT}/debug.log" when "SnapController" destination = "#{LOG_ROOT}/snapshot.log" + when "TrafficRecorder" + destination = "#{LOG_ROOT}/traffic_recorder.log" else destination = "#{LOG_ROOT}/ione.log" end msg = msg.to_s - msg = "[ #{time()} ] " + msg if _time + msg = "[ #{Time.now.ctime} ] " + msg if _time msg += " [ #{method} ]" if method != 'none' && method != "" && method != nil File.open(destination, 'a'){ |log| log.write msg + "\n" } diff --git a/service/objects/records.rb b/service/objects/records.rb deleted file mode 100644 index 397253c1..00000000 --- a/service/objects/records.rb +++ /dev/null @@ -1,29 +0,0 @@ -begin - $db.create_table :records do - primary_key :key - Integer :id, null: false - Integer :time, null: false - String :state, size: 10, null: false - end -rescue - puts "Table :records already exists, skipping" -end - -# History Record Model class -# @see https://github.com/ione-cloud/ione-sunstone/blob/55a9efd68681829624809b4895a49d750d6e6c34/ione/server/service/objects/records.rb#L1-L10 History Model Defintion -class Record < Sequel::Model(:records); end - -# States and Notifications records object(linked to VM) -class OpenNebula::Records - attr_reader :id, :records - - # @param [Fixnum] id - VM ID - def initialize id - @id = id - @records = Record.where(id:id).all # Getting records from DB[table :settings] - raise NoRecordsError if @records.empty? - end - - # No records in DB Exception - class NoRecordsError < StandardError; end -end \ No newline at end of file diff --git a/service/objects/vm.rb b/service/objects/vm.rb index 75730178..bf0da237 100644 --- a/service/objects/vm.rb +++ b/service/objects/vm.rb @@ -20,14 +20,8 @@ class OpenNebula::VirtualMachine undeploy-hard snapshot-create ) - # List of possible billing periods - # @note Read it in source by pressing button View source down - BILLING_PERIODS = [ - "0", # Pay-as-you-Go -- default - /\d/, # Number of days - ] # Generates template for OpenNebula scheduler record - def generate_schedule_str(id, action, time) + def generate_schedule_str id, action, time "\nSCHED_ACTION=[\n" + " ACTION=\"#{action}\",\n" + " ID=\"#{id}\",\n" + @@ -43,7 +37,7 @@ def schedule_actions # @param [Integer] time - Time when action schould be perfomed in secs # @param [String] periodic - Not working now # @return true - def schedule(action, time, periodic = nil) + def schedule action, time, periodic = nil return 'Unsupported action' if !SCHEDULABLE_ACTIONS.include? action self.info! id = @@ -68,7 +62,7 @@ def schedule(action, time, periodic = nil) end # Unschedules given action by ID # @note Not working, if action is already initialized - def unschedule(id) + def unschedule id self.info! schedule_data, object = self.to_hash['VM']['USER_TEMPLATE']['SCHED_ACTION'], nil @@ -97,7 +91,7 @@ def scheduler # @param [Integer] s - VM state to wait for # @param [Integer] lcm_s - VM LCM state to wait for # @return [Boolean] - def wait_for_state(s = 3, lcm_s = 3) + def wait_for_state s = 3, lcm_s = 3 i = 0 until state!() == s && lcm_state!() == lcm_s do return false if i >= 3600 @@ -125,7 +119,7 @@ def wait_for_state(s = 3, lcm_s = 3) # => 'Reconfigure Success' -- Task finished with success code, all specs are equal to given # => 'Reconfigure Unsuccessed' -- Some of specs didn't changed # => 'Reconfigure Error:{error message}' -- Exception has been generated while proceed, check your configuration - def setResourcesAllocationLimits(spec) + def setResourcesAllocationLimits spec LOG_DEBUG spec.debug_out return 'Unsupported query' if IONe.new($client, $db).get_vm_data(self.id)['IMPORTED'] == 'YES' @@ -169,7 +163,7 @@ def setResourcesAllocationLimits(spec) return nil end # Checks if vm is on given vCenter Datastore - def is_at_ds?(ds_name) + def is_at_ds? ds_name host = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)) datacenter = get_vcenter_dc(host) begin @@ -205,7 +199,7 @@ def get_vms_vcenter_ds # @option spec [String] :name VM name on vCenter node # @return [Boolean | String] # @note Method returns true if resize action ended correct, false if VM not support hot reconfiguring - def hot_resize(spec = {:name => nil}) + def hot_resize spec = {:name => nil} return false if !self.hotAddEnabled? begin host = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)) @@ -227,7 +221,7 @@ def hot_resize(spec = {:name => nil}) # @note For correct work of this method, you must keep actual vCenter Password at VCENTER_PASSWORD_ACTUAL attribute in OpenNebula # @note Method searches VM by it's default name: one-(id)-(name), if target vm got another name, you should provide it # @return [Hash | String] Returns limits Hash if success or exception message if fails - def hotAddEnabled?(name = nil) + def hotAddEnabled? name = nil begin host = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)) datacenter = get_vcenter_dc(host) @@ -246,7 +240,7 @@ def hotAddEnabled?(name = nil) # @option spec [Boolean] :ram # @option spec [String] :name VM name on vCenter node # @return [true | String] - def hotResourcesControlConf(spec = {:cpu => true, :ram => true, :name => nil}) + def hotResourcesControlConf spec = {:cpu => true, :ram => true, :name => nil} begin host, name = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)), spec[:name] datacenter = get_vcenter_dc(host) @@ -281,7 +275,7 @@ def hotResourcesControlConf(spec = {:cpu => true, :ram => true, :name => nil}) # @note For correct work of this method, you must keep actual vCenter Password at VCENTER_PASSWORD_ACTUAL attribute in OpenNebula # @note Method searches VM by it's default name: one-(id)-(name), if target vm got another name, you should provide it # @return [Hash | String] Returns limits Hash if success or exception message if fails - def getResourcesAllocationLimits(name = nil) + def getResourcesAllocationLimits name = nil begin host = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)) datacenter = get_vcenter_dc(host) @@ -299,7 +293,7 @@ def getResourcesAllocationLimits(name = nil) # Returns owner user ID # @param [Boolean] info - method doesn't get object full info one more time -- usefull if collecting data from pool # @return [Integer] - def uid(info = true) + def uid info = true self.info! if info self['UID'] end @@ -307,7 +301,7 @@ def uid(info = true) # @param [Boolean] info - method doesn't get object full info one more time -- usefull if collecting data from pool # @param [Boolean] from_pool - levels differenct between object and object received from pool.each | object | # @return [String] - def uname(info = true, from_pool = false) + def uname info = true, from_pool = false self.info! if info return @xml[0].children[3].text.to_i unless from_pool @xml.children[3].text @@ -357,210 +351,41 @@ def calculate_showback stime_req, etime_req, group_by_day = false stime = self['/VM/STIME'].to_i if self['/VM/STIME'].to_i > stime etime = self['/VM/ETIME'].to_i if self['/VM/ETIME'].to_i < etime && self['/VM/ETIME'].to_i != 0 - requested_time = (etime - stime) / 3600.0 - - ### Quick response for HOLD and PENDING vms ### - return { - "id" => id, - "name" => name, - "work_time" => 0, - "time_period_requested" => etime_req - stime_req, - "time_period_corrected" => etime - stime, - "CPU" => 0, - "MEMORY" => 0, - "DISK" => 0, - "DISK_TYPE" => self['/VM/USER_TEMPLATE/DRIVE'], - "EXCEPTION" => "State #{state == 0 ? "HOLD" : "PENDING"}", - "TOTAL" => 0 - } if state == 0 || state == 1 - - records = OpenNebula::Records.new(id).records - - if self['//BILLING_PERIOD'] == 'month' then - first = records.select{|r| r[:state] != 'pnd'}.sort_by{|r| r[:time]}.first[:time] - - unless group_by_day then - stime = Time.at(stime).to_datetime - first = Time.at(first).to_datetime - etime = Time.at(etime).to_datetime - current, periods = stime > first ? stime : first, 0 - - while current <= etime do - periods += 1 - current = current >> 1 - end + bp = self['//BILLING_PERIOD'] - return {"id" => id, "name" => name, "work_time" => etime.to_time.to_i - first.to_time.to_i, "EXCEPTION" => "Billed by calendar month", "TOTAL" => (periods * self['//PRICE'].to_f).round(2)} - end - elsif self['//BILLING_PERIOD'].to_i != 0 then + if bp.nil? || bp == 'PAYG' then + billing = Billing.new self, stime, etime + billing.make_bill + billing.receipt - first = records.select{|r| r[:state] != 'pnd'}.sort_by{|r| r[:time]}.first[:time] - delta = self['//BILLING_PERIOD'].to_i * 86400 - - unless group_by_day then - - periods = stime > first ? 0 : 1 - periods += (etime - first) / delta - - return {"id" => id, "name" => name, "work_time" => etime - first, "EXCEPTION" => "Billed per #{self['//BILLING_PERIOD'].to_i} days", "TOTAL" => (periods * self['//PRICE'].to_f).round(2)} - else - showback = [] - - diff = (Time.at(etime).to_date - Time.at(first).to_date).to_i + 1 - diff.times do | day | - showback << { - 'date' => Time.at(first + day * 86400).to_a[3..5].join('/'), - 'TOTAL' => ( - day % self['//BILLING_PERIOD'].to_i == 0 ? self['//PRICE'].to_f : 0.0 ) - } - end - return { - "id" => id, "name" => name, "work_time" => first - stime, - "EXCEPTION" => "Billed per #{self['//BILLING_PERIOD'].to_i} days", - "showback" => showback, - "TOTAL" => showback.inject(0){|total, record| total + record['TOTAL']} - } - end - end - - - - ### Generating Timeline ### - timeline = [] - records.each_with_index do | record, i | - timeline << { - 'stime' => record[:time], - 'etime' => i + 1 != records.size ? records[i + 1][:time] - 1 : etime, - 'state' => record[:state] + return { + id: id, name: name, + showback: billing.bill, + TOTAL: billing.total } - end - - timeline.delete_if { |r| (r['etime'] < stime) || (r['stime'] > etime) } - raise OpenNebula::Records::NoRecordsError if timeline.empty? - timeline[0]['stime'] = stime if timeline[0]['stime'] < stime - timeline[-1]['etime'] = etime if timeline.last['etime'] > etime - - - ### Calculating Capacity ### - cpu = self['/VM/TEMPLATE/VCPU'].to_f * self['/VM/TEMPLATE/CPU'].to_f - memory = self['/VM/TEMPLATE/MEMORY'].to_f / 1024 - disk = self['/VM/TEMPLATE/DISK/SIZE'].to_f / 1024 + elsif bp.include? 'PRE' then + curr = self['/VM/STIME'].to_i + delta = bp.split('_')[1].to_i * 86400 - ### Calculating Showback ### - cpu_cost = cpu * self['/VM/TEMPLATE/CPU_COST'].to_f - memory_cost = memory * self['/VM/TEMPLATE/MEMORY_COST'].to_f - disk_cost = disk * self['/VM/TEMPLATE/DISK_COST'].to_f + s, e = self['/VM/STIME'].to_i, self['/VM/ETIME'].to_i + total = 0 - unless group_by_day then - ### Calculating Work Time ### - work_time = 0 - timeline.each do | record | - next unless record['state'] == 'on' - work_time += record['etime'] - record['stime'] - end - work_time = work_time / 3600.0 - - - - ### Calculating Showback ### - cpu_cost *= work_time - memory_cost *= work_time - disk_cost *= requested_time + while curr < etime do + if (stime..etime).include? curr then + b = Billing.new self, curr, curr + delta + b.make_bill + b.receipt - return { - "id" => id, - "name" => name, - "work_time" => work_time, - "time_period_requested" => etime_req - stime_req, - "time_period_corrected" => etime - stime, - "CPU" => cpu_cost, - "MEMORY" => memory_cost, - "DISK" => disk_cost, - "DISK_TYPE" => self['/VM/USER_TEMPLATE/DRIVE'], - "TOTAL" => cpu_cost + memory_cost + disk_cost - } - else - timeline.clone.each_with_index do | r, i | - diff = (Time.at(r['etime']).to_date - Time.at(r['stime']).to_date).to_i - if diff >= 1 then - result, border = [], Time.at(r['stime']).to_a - border[0..2] = 59, 59, 23 - border = Time.local(*border).to_i - - result << { 'stime' => r['stime'], 'etime' => border, 'state' => r['state'], 'date' => Time.at(r['stime']).to_a[3..5].join('/') } - - (diff).times do | day | - result << { 'stime' => border += 1, 'date' => Time.at(border).to_a[3..5].join('/'), 'etime' => border += 86399, 'state' => r['state'] } - end - - result[diff]['etime'] = r['etime'] - - timeline[i] = result - else - timeline[i]['date'] = Time.at(r['stime']).to_a[3..5].join('/') + total += b.total end - end - timeline.flatten! - - timeline.map! do | r | - { - "date" => r['date'], - "requested_time" => (r['etime'] - r['stime']), - 'state' => r['state'] - } - end - timeline = timeline.group_by { | r | r['date'] } - timeline = timeline.map do | date, date_records | - result = date_records.inject({ - 'date' => date, - 'work_time' => 0, - 'requested_time' => 0, - 'CPU' => 0, - 'MEMORY' => 0, - 'DISK' => 0, - 'TOTAL' => 0 - }) do | showback, record | - requested_time = record['requested_time'] / 3600.0 - work_time = record['state'] == 'on' ? requested_time : 0 - showback['work_time'] += work_time - showback['requested_time'] += record['requested_time'] - showback['CPU'] += cpu_cost * work_time - showback['MEMORY'] += memory_cost * work_time - showback['DISK'] += disk_cost * requested_time if record['state'] != 'pnd' - - showback - end - - result['TOTAL'] += (result['CPU'] + result['MEMORY'] + result['DISK']) - result + curr += delta end return { - "id" => id, - "name" => name, - "work_time" => timeline.inject(0){|total, record| total + record['work_time']}, - "requested_time" => timeline.inject(0){|total, record| total + record['requested_time']}, - "time_period_requested" => etime_req - stime_req, - "time_period_corrected" => etime - stime, - "showback" => timeline, - "DISK_TYPE" => self['/VM/USER_TEMPLATE/DRIVE'], - "TOTAL" => timeline.inject(0){|total, record| total + record['TOTAL']} + id: id, name: name, + TOTAL: total } end - rescue OpenNebula::Records::NoRecordsError - return { - "id" => id, - "name" => name, - "work_time" => 0, - "time_period_requested" => etime_req - stime_req, - "time_period_corrected" => etime - stime, - "CPU" => 0, - "MEMORY" => 0, - "DISK" => 0, - "DISK_TYPE" => 'no_type', - "EXCEPTION" => "No Records", - "TOTAL" => 0 - } end # Returns important data in JSON format # @param [IONe] ione - IONe object for calling its methods @@ -603,4 +428,11 @@ def initialize params = [] super "#{params[0]}\nParams:#{@params.inspect}" end end + + # List VM Drives + # @return [Array] + def drives + r = to_hash!['VM']['TEMPLATE']['DISK'] + r.class == Array ? r : [ r ] + end end \ No newline at end of file diff --git a/service/on_helper.rb b/service/on_helper.rb index 4417387a..e8dd3cfc 100644 --- a/service/on_helper.rb +++ b/service/on_helper.rb @@ -1,5 +1,6 @@ require 'rbvmomi' require "#{ROOT}/service/custom_objects.rb" +require "#{ROOT}/service/showback.rb" # Useful methods for OpenNebula classes, functions and constants. module ONeHelper diff --git a/service/quotagen.rb b/service/quotagen.rb deleted file mode 100644 index be3d60bc..00000000 --- a/service/quotagen.rb +++ /dev/null @@ -1,10 +0,0 @@ -# @api private -# Generates quota template -def NewQuota(login, vmquota, cpu, memory, disk) - quota = "VM=[ - CPU=\"#{cpu}\", - MEMORY=\"#{memory}\", - SYSTEM_DISK_SIZE=\"#{disk}\", - VMS=\"#{vmquota}\" ]" - quota -end \ No newline at end of file diff --git a/service/records.rb b/service/records.rb index 558e51f1..7e6f7df9 100644 --- a/service/records.rb +++ b/service/records.rb @@ -1,11 +1,40 @@ -class OpenNebula::Records - attr_reader :id, :records +# Source of History records class +class RecordsSource + attr_reader :id - def initialize id + # VMID field key + def key + :id + end + + # @param [Fixnum] id - VM ID + def initialize cls, id @id = id - @records = $db[:records].where(id:id).to_a - raise NoRecordsError if @records.empty? + @records = cls.where(Hash[key, @id]) + end + + def records + @records.all + end + + # Find records for given time period + def find stime, etime + @records.where(time: stime..etime) + end + + def init_state stime + {} + end + + # Filter records needed for Showback Timeline + def self.tl_filter records + records + end + + # Check if source should be used with given VM + def self.check_source vm + true end +end - class NoRecordsError < StandardError; end -end \ No newline at end of file +Dir["#{ROOT}/service/records/*.rb"].each {|file| require file } diff --git a/service/records/records.rb b/service/records/records.rb new file mode 100644 index 00000000..57ead328 --- /dev/null +++ b/service/records/records.rb @@ -0,0 +1,113 @@ +begin + $db.create_table :records do + primary_key :key + Integer :id, null: false + Integer :time, null: false + String :state, size: 10, null: false + end +rescue + puts "Table :records already exists, skipping" +end + +begin + $db.create_table :snapshot_records do + primary_key :key + foreign_key :vm, :vm_pool, null: false + Integer :id, null: false + Integer :crt, null: false + Integer :del, null: true + end +rescue + puts "Table :snapshot_records already exists, skipping" +end + +# History Record Model class +# @see https://github.com/ione-cloud/ione-sunstone/blob/55a9efd68681829624809b4895a49d750d6e6c34/ione/server/service/objects/records.rb#L1-L10 History Model Defintion +class Record < Sequel::Model(:records) + def sortable + self + end + + def sorter + time + end + alias :ts :sorter + + def mod st + st[:state] = state + end +end + +class SnapshotRecord < Sequel::Model(:snapshot_records) + + class CreateSnapshotRecord < SnapshotRecord + def sorter + crt + end + alias :ts :sorter + + def mod st + st[:snaps] += 1 + end + end + class DeleteSnapshotRecord < SnapshotRecord + def sorter + del + end + alias :ts :sorter + + def mod st + st[:snaps] -= 1 + end + end + + def values + @values.without(:key) + end + + def sortable + if self.del then + [ CreateSnapshotRecord.new(values), DeleteSnapshotRecord.new(values) ] + else + CreateSnapshotRecord.new(values) + end + end +end + +class OpenNebula::Records < RecordsSource + def initialize id + super(Record, id) + end + def init_state stime + prev = @records.where{ time < stime }.order(Sequel.desc :time).limit(1).to_a.last + if prev.nil? then + { + 'state': @records.where{ time >= stime}.limit(1).to_a.first.state + } + else + { + state: prev.state + } + end + end +end +class OpenNebula::SnapshotRecords < RecordsSource + + def key + :vm + end + + def initialize id + super(SnapshotRecord, id) + end + def find stime, etime + @records.where(crt: stime..etime).or(del: stime..etime) + end + + def init_state stime + # SELECT * FROM `snapshot_records` WHERE ((`crt` < 0) AND ((`del` >= 0) OR NOT `del`)) + { + snaps: @records.where{ crt < stime }.where{(del >= stime) | ~del}.count + } + end +end diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb new file mode 100644 index 00000000..ccccad3d --- /dev/null +++ b/service/records/traffic_records.rb @@ -0,0 +1,131 @@ +begin + $db.create_table :traffic_records do + primary_key :key + foreign_key :vm, :vm_pool, null: false + String :rx, null: false + String :rx_last, null: false + String :tx, null: false + String :tx_last, null: false + Integer :stime, null: false + Integer :etime + end +rescue + puts "Table :traffic_records already exists, skipping" +end + +class TrafficRecord < Sequel::Model(:traffic_records) + def sortable + self + end + def sorter + etime + end + alias :ts :sorter + + def mod st + st.merge! rx: rx.to_i, tx: tx.to_i + end + + def conv_keys + [:rx, :tx, :rx_last, :tx_last] + end + def to_i + for key in conv_keys do + @values[key] = @values[key].to_i + end + self + end + def to_s + for key in conv_keys do + @values[key] = @values[key].to_s + end + self + end +end + +class OpenNebula::TrafficRecords < RecordsSource + + def key + :vm + end + + def initialize id, nosync = false + super(TrafficRecord, id) + sync unless nosync + end + + def sync vm = nil + if vm.nil? then + vm = onblock :vm, @id + vm.info! + end + + last = TrafficRecord.where(vm: vm.id).order(Sequel.asc(:stime)).last + if last.nil? then + return 0 + end + last = last.to_i + + def next? ts, last # Filter expression + return !last[:etime].nil? && ts <= last[:etime] + end + + mon_raw = vm.monitoring(['NETTX', 'NETRX']) + mon = {} + mon_raw['NETTX'].each do | el | + el[0] = el[0].to_i + next if next? el.first, last # Filter records which have been counted + mon[el.first] = {} + mon[el.first][:tx] = el.last + end + mon_raw['NETRX'].each do | el | + el[0] = el[0].to_i + next if next? el.first, last # Filter records which have been counted + mon[el.first][:rx] = el.last + end + + return 0 if mon.keys.size == 0 + + for ts, data in mon do + data[:rx], data[:tx] = data[:rx].to_i, data[:tx].to_i + last = last.to_i + + if last[:rx_last] > data[:rx] || last[:tx_last] > data[:tx] then + last[:rx] += data[:rx] + last[:tx] += data[:tx] + else + last[:rx] += (data[:rx] - last[:rx_last]) + last[:tx] += (data[:tx] - last[:tx_last]) + end + + last[:rx_last], last[:tx_last] = data[:rx], data[:tx] + last[:etime] = ts + end + + last = last.to_s + TrafficRecord.where(stime: last[:stime]).update(**last.values) + + mon.keys.size + end + + def find st, et + last = TrafficRecord.where(vm: @id).order(Sequel.asc(:stime)).last + if last[:etime] - last[:stime] >= 86400 then # If record is elder than 24 hours + args = last.values.without(:key, :rx, :tx, :stime) + args.merge! rx: 0, tx: 0, stime: args[:etime] # Setting up new record with zero rx, tx and same rx_last, tx_last + TrafficRecord.insert(**args.without(:etime)) + end + + @records.exclude(etime: nil).exclude{ etime - stime < 86400 }.where(etime: st..et) + end + + def init_state stime + state = { rx: 0, tx: 0 } + rec = TrafficRecord.where(vm: @id).where(etime: stime).all.last + unless rec.nil? then + rec = rec.to_i + state[:rx], state[:tx] = rec.rx, rec.tx + end + state + end +end \ No newline at end of file diff --git a/service/showback.rb b/service/showback.rb new file mode 100644 index 00000000..0d44c0db --- /dev/null +++ b/service/showback.rb @@ -0,0 +1,112 @@ +require "#{ROOT}/service/records.rb" +require "#{ROOT}/service/biller.rb" + +# Class for compiling all history records from all records sources into one readable timeline +class Timeline + + attr_reader :vm, :stime, :etime, :group_by_day, :timeline, :sources, :compiled, :state + + SOURCES = [ + Records, SnapshotRecords, TrafficRecords + ] + + def initialize vm, stime, etime, group_by_day = false + @vm, @stime, @etime, @group_by_day = vm, stime, etime, group_by_day + @sources = SOURCES + @compiled = false + end + + def compile + records = @sources.inject([]) do | r, source | + r.concat source.tl_filter( + source.new(@vm.id).find(@stime, @etime).all + ) + end + + records.map! do | rec | + rec.sortable + end + records.flatten! + + @timeline = records.sort_by { |rec| rec.sorter } + @timeline.select! { |rec| rec.sorter.between?(@stime, @etime)} + + init + init_rec = InitRecord.new @stime, @state + @timeline.unshift init_rec + @timeline << FinalRecord.new(@etime) + + @compiled = true + self + end + + def init + @state = @sources.inject({}) do | r, source | + r.merge source.new(@vm.id).init_state(@stime) + end + end + + class InitRecord + def initialize time, state + @time, @state = time, state + end + def ts + @time + end + def mod st + st.merge! @state + end + end + class FinalRecord + def initialize time + @time = time + end + def ts + @time + end + end +end + +# Class for billing through Timeline using different billers +class Billing + + attr_reader :timeline, :bill + + BILLERS = [ + CapacityBiller, DiskBiller, SnapshotsBiller, TrafficBiller + ] + + def initialize vm, stime, etime + @vm = vm + @billers = BILLERS.map { | bill | bill.new(@vm) } + @billers.select! { |bill| bill.check_biller } + + @timeline = Timeline.new vm, stime, etime + @timeline.compile + end + + def make_bill + state, @bill = {}, [] + @timeline.timeline.each_cons(2) do | curr, con | + delta = con.ts - curr.ts + curr.mod state + bill_rec = {time: con.ts} + @billers.each do | biller | + bill_rec.merge! biller.bill(bill: bill_rec, state: state, delta: delta, record: curr) + end + @bill << bill_rec + end + + @bill + end + + def receipt + @bill.map! do | el | + el.merge total: el.without(:time).values.sum + end + end + + def total + @bill.inject(0) { |r, el| r += el[:total] } + end +end \ No newline at end of file diff --git a/service/time.rb b/service/time.rb index bbb6e33e..a4147034 100644 --- a/service/time.rb +++ b/service/time.rb @@ -1,9 +1,3 @@ -# Returns current time in ctime format -# @return [String] -def time - Time.now.ctime -end - # Formats time from seconds from the start of Time to dd:hh:mm:ss format # @param [Integer] sec # @return [String] diff --git a/sys/ione.conf b/sys/ione.conf index e36cf249..18fc75ba 100644 --- a/sys/ione.conf +++ b/sys/ione.conf @@ -37,6 +37,9 @@ vCenter: SnapshotController: check-period: 3600 # Snapshots check frequency in seconds +TrafficRecorder: + check-period: 3600 # Traffic monitoring check frequency in seconds + DB: # This values are available in /etc/one/oned.conf user: user pass: secret