From f36429492c5bcb6aa1aaf16bf62ef90fa49f351e Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 2 Dec 2020 16:33:06 +0100 Subject: [PATCH 01/87] snapshots record class --- service/objects/records.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/service/objects/records.rb b/service/objects/records.rb index 397253c1..504c0ab4 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -9,10 +9,23 @@ puts "Table :records already exists, skipping" end +begin + $db.create_table :snapshot_records do + primary_key :key + foreign_key :vm, :vm_pool, null: false + Integer :id, null: false + String :action, size: 3, null: false + end +rescue + puts "Table :snapshot_records already exists, skipping" +end + # History Record Model class # @see https://github.com/ione-cloud/ione-sunstone/blob/55a9efd68681829624809b4895a49d750d6e6c34/ione/server/service/objects/records.rb#L1-L10 History Model Defintion class Record < Sequel::Model(:records); end +class SnapshotRecord < Sequel::Model(:snapshot_records); end + # States and Notifications records object(linked to VM) class OpenNebula::Records attr_reader :id, :records From c49cc17fe010e5ab2750bd572e2bfc03448337c0 Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 2 Dec 2020 16:33:47 +0100 Subject: [PATCH 02/87] deprecated --- service/handlers/cache_handler.rb | 44 ------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 service/handlers/cache_handler.rb diff --git a/service/handlers/cache_handler.rb b/service/handlers/cache_handler.rb deleted file mode 100644 index 78b670ba..00000000 --- a/service/handlers/cache_handler.rb +++ /dev/null @@ -1,44 +0,0 @@ -# @!visibility private -class CacheStack - - include Enumerable - # Initializer of CacheStack object - def initialize(num = 5) - @size = num - @queue = Array.new - end - # Stack iterator - def each(&blk) - @queue.each(&blk) - end - # Stack pop method - def pop - @queue.pop - end - # Stack push method - def push(value) - @queue.shift if @queue.size >= @size - @queue.push(value) - end - # Stack as array - def to_a - @queue.to_a - end - # Alias for push - def <<(value) - push(value) - end - # Reads last object - def last - return @queue.last - end - # Gets object if include - def get_if_include(data) - return (self << @queue.delete(data)).last - end - # Returns string array of Stack - def to_s - return @queue - end - -end \ No newline at end of file From 52adc823882224eae7944f6eaf6635364e6b276c Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 2 Dec 2020 16:34:19 +0100 Subject: [PATCH 03/87] deprecated --- service/quotagen.rb | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 service/quotagen.rb diff --git a/service/quotagen.rb b/service/quotagen.rb deleted file mode 100644 index be3d60bc..00000000 --- a/service/quotagen.rb +++ /dev/null @@ -1,10 +0,0 @@ -# @api private -# Generates quota template -def NewQuota(login, vmquota, cpu, memory, disk) - quota = "VM=[ - CPU=\"#{cpu}\", - MEMORY=\"#{memory}\", - SYSTEM_DISK_SIZE=\"#{disk}\", - VMS=\"#{vmquota}\" ]" - quota -end \ No newline at end of file From 6ac689a755133e49db14fd90ee0d873ed183b7fe Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 2 Dec 2020 16:35:16 +0100 Subject: [PATCH 04/87] deprecated --- service/time.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/service/time.rb b/service/time.rb index bbb6e33e..a4147034 100644 --- a/service/time.rb +++ b/service/time.rb @@ -1,9 +1,3 @@ -# Returns current time in ctime format -# @return [String] -def time - Time.now.ctime -end - # Formats time from seconds from the start of Time to dd:hh:mm:ss format # @param [Integer] sec # @return [String] From de57dcf8858cae59924640e06bea9792f30b45f1 Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 2 Dec 2020 16:36:04 +0100 Subject: [PATCH 05/87] deprecated --- service/records.rb | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 service/records.rb diff --git a/service/records.rb b/service/records.rb deleted file mode 100644 index 558e51f1..00000000 --- a/service/records.rb +++ /dev/null @@ -1,11 +0,0 @@ -class OpenNebula::Records - attr_reader :id, :records - - def initialize id - @id = id - @records = $db[:records].where(id:id).to_a - raise NoRecordsError if @records.empty? - end - - class NoRecordsError < StandardError; end -end \ No newline at end of file From e44616b9de113ebf2975c49a73515e72df39a5ad Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 2 Dec 2020 16:53:38 +0100 Subject: [PATCH 06/87] deprecated --- service/log.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/service/log.rb b/service/log.rb index b75cecbe..5becfa5b 100644 --- a/service/log.rb +++ b/service/log.rb @@ -35,7 +35,7 @@ def LOG(msg, method = "none", _time = true) destination = "#{LOG_ROOT}/ione.log" end msg = msg.to_s - msg = "[ #{time()} ] " + msg if _time + msg = "[ #{Time.now.ctime} ] " + msg if _time msg += " [ #{method} ]" if method != 'none' && method != "" && method != nil File.open(destination, 'a'){ |log| log.write msg + "\n" } @@ -50,7 +50,7 @@ def LOG_COLOR(msg, method = caller_locations(1,1)[0].label.dup, color = 'red', f destination = "#{LOG_ROOT}/ione.log" destination = "#{LOG_ROOT}/snapshot.log" if method == "SnapController" msg = msg.to_s.send(color).send(font) - msg = "[ #{time()} ] " + msg + msg = "[ #{Time.now.ctime} ] " + msg method.slice!('block in '.dup) msg += " [ #{method} ]" if method != 'none' && method != "" && method != nil @@ -64,7 +64,7 @@ def LOG_COLOR(msg, method = caller_locations(1,1)[0].label.dup, color = 'red', f # Logging the message directly into LOG_LOCATION/debug.log def LOG_DEBUG(msg, method = 'DEBUG', _time = true) destination = "#{LOG_ROOT}/debug.log" - msg = "[ #{time()} ] #{msg}" + msg = "[ #{Time.now.ctime} ] #{msg}" File.open(destination, 'a'){ |log| log.write msg + "\n" } $log << "#{msg} | #{destination}" true @@ -87,7 +87,7 @@ def LOG_AUTO(msg, method = caller_locations(1,1)[0].label, _time = true) destination = "#{LOG_ROOT}/ione.log" end msg = msg.to_s - msg = "[ #{time()} ] " + msg if _time + msg = "[ #{Time.now.ctime} ] " + msg if _time msg += " [ #{method} ]" if method != 'none' && method != "" && method != nil File.open(destination, 'a'){ |log| log.write msg + "\n" } From fbd628c3d7fa477367274e3ef843359c2b171982 Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 2 Dec 2020 18:08:36 +0100 Subject: [PATCH 07/87] Records Source Base --- service/objects/records.rb | 46 ++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/service/objects/records.rb b/service/objects/records.rb index 504c0ab4..57288605 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -12,9 +12,10 @@ begin $db.create_table :snapshot_records do primary_key :key - foreign_key :vm, :vm_pool, null: false - Integer :id, null: false - String :action, size: 3, null: false + foreign_key :vm, :vm_pool, null: false + Integer :id, null: false + Integer :time, null: false + String :action, size: 3, null: false end rescue puts "Table :snapshot_records already exists, skipping" @@ -26,17 +27,38 @@ class Record < Sequel::Model(:records); end class SnapshotRecord < Sequel::Model(:snapshot_records); end -# States and Notifications records object(linked to VM) -class OpenNebula::Records - attr_reader :id, :records +class RecordsSource + attr_reader :id + + @@key = :id + @@time_delimeter_col = :time # @param [Fixnum] id - VM ID - def initialize id + def initialize cls, id @id = id - @records = Record.where(id:id).all # Getting records from DB[table :settings] - raise NoRecordsError if @records.empty? + @records = cls.where(Hash[@@key, @id]) + end + + def records + @records.all + end + + def find stime, etime + @records.where(Hash[@@time_delimeter_col, stime..etime]) + end +end + +class OpenNebula::Records < RecordsSource + def initialize id + super(Record, id) + end +end +class OpenNebula::SnapshotRecords < RecordsSource + + @@key = :vm + + def initialize id + super(SnapshotRecord, id) end +end - # No records in DB Exception - class NoRecordsError < StandardError; end -end \ No newline at end of file From 4559081e59e8cf2d90d4809f7e576dd23175a377 Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 2 Dec 2020 18:18:50 +0100 Subject: [PATCH 08/87] timeline filter method --- service/objects/records.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service/objects/records.rb b/service/objects/records.rb index 57288605..e344d113 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -46,6 +46,10 @@ def records def find stime, etime @records.where(Hash[@@time_delimeter_col, stime..etime]) end + + def self.tl_filter records + records + end end class OpenNebula::Records < RecordsSource From c137166085fddec869f9b659d5f8beb376cf482f Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 2 Dec 2020 18:29:46 +0100 Subject: [PATCH 09/87] different snapshot records scheme --- service/objects/records.rb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/service/objects/records.rb b/service/objects/records.rb index e344d113..03877a31 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -12,10 +12,10 @@ begin $db.create_table :snapshot_records do primary_key :key - foreign_key :vm, :vm_pool, null: false - Integer :id, null: false - Integer :time, null: false - String :action, size: 3, null: false + foreign_key :vm, :vm_pool, null: false + Integer :id, null: false + Integer :crt, null: false + Integer :del, null: true end rescue puts "Table :snapshot_records already exists, skipping" @@ -64,5 +64,11 @@ class OpenNebula::SnapshotRecords < RecordsSource def initialize id super(SnapshotRecord, id) end + def find stime, etime + @records.where{ + (((del == nil) | (del >= etime)) & (crt <= stime)) || + (crt =~ (stime..etime)) + } + end end From 69b017469e9b47f0437c27ac91c35b80fb36d8c9 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 12:33:02 +0100 Subject: [PATCH 10/87] correct key management --- service/objects/records.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/service/objects/records.rb b/service/objects/records.rb index 03877a31..b74881cd 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -30,13 +30,16 @@ class SnapshotRecord < Sequel::Model(:snapshot_records); end class RecordsSource attr_reader :id - @@key = :id + def key + :id + end + @@time_delimeter_col = :time # @param [Fixnum] id - VM ID def initialize cls, id @id = id - @records = cls.where(Hash[@@key, @id]) + @records = cls.where(Hash[key, @id]) end def records @@ -59,7 +62,9 @@ def initialize id end class OpenNebula::SnapshotRecords < RecordsSource - @@key = :vm + def key + :vm + end def initialize id super(SnapshotRecord, id) From 629854f5cd7d50ba9075c3315a35bfca183146ef Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 12:34:01 +0100 Subject: [PATCH 11/87] sortable method --- service/objects/records.rb | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/service/objects/records.rb b/service/objects/records.rb index b74881cd..6d94bbbc 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -23,9 +23,28 @@ # History Record Model class # @see https://github.com/ione-cloud/ione-sunstone/blob/55a9efd68681829624809b4895a49d750d6e6c34/ione/server/service/objects/records.rb#L1-L10 History Model Defintion -class Record < Sequel::Model(:records); end +class Record < Sequel::Model(:records) + def sortable + self + end -class SnapshotRecord < Sequel::Model(:snapshot_records); end + class CreateSnapshotRecord < SnapshotRecord + end + class DeleteSnapshotRecord < SnapshotRecord + end + + def values + @values.without(:key) + end + + def sortable + if self.del then + [ CreateSnapshotRecord.new(values), DeleteSnapshotRecord.new(values) ] + else + CreateSnapshotRecord.new(values) + end + end +end class RecordsSource attr_reader :id From 0ecb00886bdae92bd6d3edfd8a13626d4656d2d7 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 12:34:16 +0100 Subject: [PATCH 12/87] sorter method --- service/objects/records.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/service/objects/records.rb b/service/objects/records.rb index 6d94bbbc..1fcd4699 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -28,9 +28,22 @@ def sortable self end + def sorter + time + end +end + +class SnapshotRecord < Sequel::Model(:snapshot_records) + class CreateSnapshotRecord < SnapshotRecord + def sorter + crt + end end class DeleteSnapshotRecord < SnapshotRecord + def sorter + del + end end def values From 760d6c34f7000cbc1dafdd6c48ff20f2253c7b00 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 12:34:54 +0100 Subject: [PATCH 13/87] Timeline base class --- service/showback.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 service/showback.rb diff --git a/service/showback.rb b/service/showback.rb new file mode 100644 index 00000000..fdd447e1 --- /dev/null +++ b/service/showback.rb @@ -0,0 +1,29 @@ +class Timeline + + attr_reader :vmid, :stime, :etime, :group_by_day, :timeline + + SOURCES = [ + Records, SnapshotRecords + ] + + def initialize vmid, stime, etime, group_by_day = false + @vmid, @stime, @etime, @group_by_day = vmid, stime, etime, group_by_day + end + + def compile + records = SOURCES.inject([]) do | r, source | + r.concat source.tl_filter( + source.new(@vmid).find(@stime, @etime).all + ) + end + + records.map! do | rec | + rec.sortable + end + records.flatten! + + @timeline = records.sort_by { |rec| rec.sorter } + + self + end +end From d2c3f5b338492845cbeb6cf688d8c8edb0ee7fc6 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:08:03 +0100 Subject: [PATCH 14/87] sync --- service/objects/records.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/service/objects/records.rb b/service/objects/records.rb index 1fcd4699..3d5ecac4 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -108,4 +108,3 @@ def find stime, etime } end end - From 2340168cd45ce5251082022c11c2cc228d6fa7b8 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:11:16 +0100 Subject: [PATCH 15/87] vm instead of vm.id and personal sources --- service/showback.rb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/service/showback.rb b/service/showback.rb index fdd447e1..9bcc8527 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -6,14 +6,20 @@ class Timeline Records, SnapshotRecords ] - def initialize vmid, stime, etime, group_by_day = false - @vmid, @stime, @etime, @group_by_day = vmid, stime, etime, group_by_day + def initialize vm, stime, etime, group_by_day = false + @vm, @stime, @etime, @group_by_day = vm, stime, etime, group_by_day + + @sources = SOURCES + end + + def set_state state + @state = state end def compile - records = SOURCES.inject([]) do | r, source | + records = @sources.inject([]) do | r, source | r.concat source.tl_filter( - source.new(@vmid).find(@stime, @etime).all + source.new(@vm.id).find(@stime, @etime).all ) end From 36e9ff3bea154b1cc4868136f89fa8c6faf1708e Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:14:34 +0100 Subject: [PATCH 16/87] docs and deprecated var --- service/objects/records.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/service/objects/records.rb b/service/objects/records.rb index 3d5ecac4..23c0b0e7 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -59,15 +59,15 @@ def sortable end end +# Source of History records class class RecordsSource attr_reader :id + # VMID field key def key :id end - @@time_delimeter_col = :time - # @param [Fixnum] id - VM ID def initialize cls, id @id = id @@ -78,13 +78,20 @@ def records @records.all end + # Find records for given time period def find stime, etime - @records.where(Hash[@@time_delimeter_col, stime..etime]) + @records.where(Hash[:time, stime..etime]) end + # Filter records needed for Showback Timeline def self.tl_filter records records end + + # Check if source should be used with given VM + def self.check_source vm + true + end end class OpenNebula::Records < RecordsSource From 2ff52302e9b54cf1698c7e660aaa779d4ff87bdc Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:49:42 +0100 Subject: [PATCH 17/87] Biller base class --- service/biller.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 service/biller.rb diff --git a/service/biller.rb b/service/biller.rb new file mode 100644 index 00000000..5a653d94 --- /dev/null +++ b/service/biller.rb @@ -0,0 +1,16 @@ +class Biller + + def initialize vm + @vm = vm + end + + def costs + SETTINGS_TABLE.as_hash(:name, :body).select {|key| key.include? 'COST' } + end + + def check_biller + true + end +end + +Dir["#{ROOT}/service/billers/*.rb"].each {|file| require file } From f69a28ba92bbc2683bbaf29a991e6ad975f05263 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:50:22 +0100 Subject: [PATCH 18/87] moving billing to Billing class --- service/showback.rb | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/service/showback.rb b/service/showback.rb index 9bcc8527..25dad9b0 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -1,6 +1,8 @@ +require "#{ROOT}/service/biller.rb" + class Timeline - attr_reader :vmid, :stime, :etime, :group_by_day, :timeline + attr_reader :vm, :stime, :etime, :group_by_day, :timeline, :sources, :compiled SOURCES = [ Records, SnapshotRecords @@ -8,12 +10,8 @@ class Timeline def initialize vm, stime, etime, group_by_day = false @vm, @stime, @etime, @group_by_day = vm, stime, etime, group_by_day - @sources = SOURCES - end - - def set_state state - @state = state + @compiled = false end def compile @@ -29,6 +27,30 @@ def compile records.flatten! @timeline = records.sort_by { |rec| rec.sorter } + @compiled = true + self + end +end + +class Billing + BILLERS = [ + CapacityBiller + ] + + def initialize vm, stime, etime + @vm = vm + @billers = BILLERS.map { | bill | bill.new(vm) } + @billers.select! { |bill| bill.check_biller } + + @timeline = Timeline.new vm, stime, etime + @timeline.compile + end + + + def set_state state + @state = state + end +end self end From 9c3e3028ab670213d613e7137c5b26bf379bc756 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:51:23 +0100 Subject: [PATCH 19/87] comm --- service/showback.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/service/showback.rb b/service/showback.rb index 25dad9b0..61603c2b 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -1,5 +1,6 @@ require "#{ROOT}/service/biller.rb" +# Class for compiling all history records from all records sources into one readable timeline class Timeline attr_reader :vm, :stime, :etime, :group_by_day, :timeline, :sources, :compiled @@ -32,6 +33,7 @@ def compile end end +# Class for billing through Timeline using different billers class Billing BILLERS = [ CapacityBiller @@ -51,7 +53,7 @@ def set_state state @state = state end end - + self end end From 6c20a402f60acc8d242c881780b0bea5b6135565 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:51:49 +0100 Subject: [PATCH 20/87] including showback --- service/on_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/service/on_helper.rb b/service/on_helper.rb index 4417387a..e8dd3cfc 100644 --- a/service/on_helper.rb +++ b/service/on_helper.rb @@ -1,5 +1,6 @@ require 'rbvmomi' require "#{ROOT}/service/custom_objects.rb" +require "#{ROOT}/service/showback.rb" # Useful methods for OpenNebula classes, functions and constants. module ONeHelper From ca0e45769b17cc4e421fb3f051aa73ae6ddfb0e5 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:51:59 +0100 Subject: [PATCH 21/87] capacity biller base --- service/billers/capacity.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 service/billers/capacity.rb diff --git a/service/billers/capacity.rb b/service/billers/capacity.rb new file mode 100644 index 00000000..ea4334a9 --- /dev/null +++ b/service/billers/capacity.rb @@ -0,0 +1,16 @@ +class CapacityBiller < Biller + def check_biller + @costs = JSON.parse(costs['CAPACITY_COST']) + return false if @costs.nil? + + r = + @costs.values.inject(0) do | r, c | + r += c.to_i + rescue + r + end + return r > 0 + rescue + return false + end +end \ No newline at end of file From 50d90b6b6d2dc1b34b58a4c7d0f1f6cb6ef1bc1e Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:52:29 +0100 Subject: [PATCH 22/87] comm --- service/biller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/biller.rb b/service/biller.rb index 5a653d94..676e1e3f 100644 --- a/service/biller.rb +++ b/service/biller.rb @@ -4,10 +4,12 @@ def initialize vm @vm = vm end + # Costs hash def costs SETTINGS_TABLE.as_hash(:name, :body).select {|key| key.include? 'COST' } end + # Check if this biller should be used in Billing def check_biller true end From 11bed433b0ebe91d8ba1c0eed0cff7e7adf451ef Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:53:05 +0100 Subject: [PATCH 23/87] comm --- service/billers/capacity.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/service/billers/capacity.rb b/service/billers/capacity.rb index ea4334a9..55d20f25 100644 --- a/service/billers/capacity.rb +++ b/service/billers/capacity.rb @@ -1,4 +1,5 @@ class CapacityBiller < Biller + # Checking if Capacity costs are given, otherwise there is no point to calculate it def check_biller @costs = JSON.parse(costs['CAPACITY_COST']) return false if @costs.nil? From 1f9977c543d65b25b4b6cf55e79a6adb92f046d9 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 13:54:33 +0100 Subject: [PATCH 24/87] DiskBiller base --- service/billers/disk.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 service/billers/disk.rb diff --git a/service/billers/disk.rb b/service/billers/disk.rb new file mode 100644 index 00000000..15a50d94 --- /dev/null +++ b/service/billers/disk.rb @@ -0,0 +1,17 @@ +class DiskBiller < Biller + # Checking if Capacity costs are given, otherwise there is no point to calculate it + def check_biller + @costs = JSON.parse(costs['DISK_COST']) + return false if @costs.nil? + + r = + @costs.values.inject(0) do | r, c | + r += c.to_i + rescue + r + end + return r > 0 + rescue + return false + end +end \ No newline at end of file From 5249789e3b0242b5490b9652cddc45ced4084128 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 15:19:59 +0100 Subject: [PATCH 25/87] source init_state method --- service/objects/records.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/service/objects/records.rb b/service/objects/records.rb index 23c0b0e7..10dbeead 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -83,6 +83,10 @@ def find stime, etime @records.where(Hash[:time, stime..etime]) end + def init_state stime + {} + end + # Filter records needed for Showback Timeline def self.tl_filter records records @@ -98,6 +102,18 @@ class OpenNebula::Records < RecordsSource def initialize id super(Record, id) end + def init_state stime + prev = @records.where{ time < stime }.order(Sequel.desc :time).limit(1).to_a.last + if prev.nil? then + { + 'state': @records.where{ time >= stime}.limit(1).to_a.first.state + } + else + { + state: prev.state + } + end + end end class OpenNebula::SnapshotRecords < RecordsSource From b56dd2df0208a1b8eccd8b7dc7b207d34fbba5de Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 15:20:08 +0100 Subject: [PATCH 26/87] rudimentaar --- service/objects/records.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/objects/records.rb b/service/objects/records.rb index 10dbeead..853518dd 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -80,7 +80,7 @@ def records # Find records for given time period def find stime, etime - @records.where(Hash[:time, stime..etime]) + @records.where(time: stime..etime) end def init_state stime From b1821f0bbd7bb685c622053f0dd41e02411097dc Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 15:20:26 +0100 Subject: [PATCH 27/87] kiss find method --- service/objects/records.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/service/objects/records.rb b/service/objects/records.rb index 853518dd..3da55bc9 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -125,9 +125,9 @@ def initialize id super(SnapshotRecord, id) end def find stime, etime - @records.where{ - (((del == nil) | (del >= etime)) & (crt <= stime)) || - (crt =~ (stime..etime)) + @records.where(crt: stime..etime).or(del: stime..etime) + end + } end end From 8aa7015fc98ed4b471be6030de9672806ca12bde Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 15:24:32 +0100 Subject: [PATCH 28/87] SnapshotRecordsSource init_state --- service/objects/records.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service/objects/records.rb b/service/objects/records.rb index 3da55bc9..7fe4b94f 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -128,6 +128,10 @@ def find stime, etime @records.where(crt: stime..etime).or(del: stime..etime) end + def init_state stime + # SELECT * FROM `snapshot_records` WHERE ((`crt` < 0) AND ((`del` >= 0) OR NOT `del`)) + { + snaps: @records.where{ crt < stime }.where{(del >= stime) | ~del}.count } end end From 20493f52440262d9526ff8b1be1843964cdd34bc Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 15:26:18 +0100 Subject: [PATCH 29/87] init method for collecting initial vm state --- service/showback.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/service/showback.rb b/service/showback.rb index 61603c2b..f087fa7e 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -3,7 +3,7 @@ # Class for compiling all history records from all records sources into one readable timeline class Timeline - attr_reader :vm, :stime, :etime, :group_by_day, :timeline, :sources, :compiled + attr_reader :vm, :stime, :etime, :group_by_day, :timeline, :sources, :compiled, :state SOURCES = [ Records, SnapshotRecords @@ -28,9 +28,16 @@ def compile records.flatten! @timeline = records.sort_by { |rec| rec.sorter } + @timeline.select! { |rec| rec.sorter.between?(stime, etime)} @compiled = true self end + + def init + @state = @sources.inject({}) do | r, source | + r.merge source.new(@vm.id).init_state(@stime) + end + end end # Class for billing through Timeline using different billers From d7616c430f47c48ccc9cd19140e28b1956d75650 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 15:26:35 +0100 Subject: [PATCH 30/87] initiate timeline on billing --- service/showback.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service/showback.rb b/service/showback.rb index f087fa7e..7888bbe9 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -42,6 +42,9 @@ def init # Class for billing through Timeline using different billers class Billing + + attr_reader :timeline + BILLERS = [ CapacityBiller ] @@ -53,6 +56,7 @@ def initialize vm, stime, etime @timeline = Timeline.new vm, stime, etime @timeline.compile + @timeline.init end From b20ab42d7a27fdb251393f9586f82228941c0181 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 4 Dec 2020 17:30:52 +0100 Subject: [PATCH 31/87] sorter alias --- service/objects/records.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/service/objects/records.rb b/service/objects/records.rb index 7fe4b94f..9ea28495 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -31,6 +31,7 @@ def sortable def sorter time end + alias :ts :sorter end class SnapshotRecord < Sequel::Model(:snapshot_records) @@ -39,11 +40,13 @@ class CreateSnapshotRecord < SnapshotRecord def sorter crt end + alias :ts :sorter end class DeleteSnapshotRecord < SnapshotRecord def sorter del end + alias :ts :sorter end def values From a03255138b2a9a62dedb711e7bf49a2961eb3bb7 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:22:54 +0100 Subject: [PATCH 32/87] Using new showback engine --- ione_server.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ione_server.rb b/ione_server.rb index 7661f6e0..1a0a85f3 100644 --- a/ione_server.rb +++ b/ione_server.rb @@ -115,6 +115,8 @@ class Settings < Sequel::Model(:settings); end puts 'Including on_helper funcs' require "#{ROOT}/service/on_helper.rb" include ONeHelper +puts 'Including showback' +require "#{ROOT}/service/showback.rb" puts 'Including Deferable module' require "#{ROOT}/service/defer.rb" From a10c2a9a8d4a2a70a5327fff72655a8e198bb463 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:23:24 +0100 Subject: [PATCH 33/87] store capacity cost on check --- service/billers/capacity.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/service/billers/capacity.rb b/service/billers/capacity.rb index 55d20f25..9c7b02c9 100644 --- a/service/billers/capacity.rb +++ b/service/billers/capacity.rb @@ -4,13 +4,13 @@ def check_biller @costs = JSON.parse(costs['CAPACITY_COST']) return false if @costs.nil? - r = + @cost = @costs.values.inject(0) do | r, c | - r += c.to_i + r += c.to_f rescue r end - return r > 0 + return @cost > 0 rescue return false end From 6923200197cb8621b15d14b7acb7e39c2660b132 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:24:03 +0100 Subject: [PATCH 34/87] bill method for Biller --- service/biller.rb | 4 ++++ service/billers/capacity.rb | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/service/biller.rb b/service/biller.rb index 676e1e3f..c0d6bf75 100644 --- a/service/biller.rb +++ b/service/biller.rb @@ -13,6 +13,10 @@ def costs def check_biller true end + + def bill bill, state, delta + 0 + end end Dir["#{ROOT}/service/billers/*.rb"].each {|file| require file } diff --git a/service/billers/capacity.rb b/service/billers/capacity.rb index 9c7b02c9..0195d98e 100644 --- a/service/billers/capacity.rb +++ b/service/billers/capacity.rb @@ -14,4 +14,11 @@ def check_biller rescue return false end + + def bill bill, state, delta + if state[:state] == 'on' then + bill[:capacity] = delta * @cost + end + bill + end end \ No newline at end of file From 7082932f8ef3b59a4b97d36f9c13866157ea02b4 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:24:40 +0100 Subject: [PATCH 35/87] modify state method for Records --- service/objects/records.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/service/objects/records.rb b/service/objects/records.rb index 9ea28495..a43a8f6a 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -32,6 +32,10 @@ def sorter time end alias :ts :sorter + + def mod st + st[:state] = state + end end class SnapshotRecord < Sequel::Model(:snapshot_records) @@ -41,12 +45,20 @@ def sorter crt end alias :ts :sorter + + def mod st + st[:snaps] += 1 + end end class DeleteSnapshotRecord < SnapshotRecord def sorter del end alias :ts :sorter + + def mod st + st[:snaps] -= 1 + end end def values From 2eb94f1aa076e807953d197b24934a245b972f55 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:24:54 +0100 Subject: [PATCH 36/87] corr attr --- service/showback.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/showback.rb b/service/showback.rb index 7888bbe9..d778087f 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -51,7 +51,7 @@ class Billing def initialize vm, stime, etime @vm = vm - @billers = BILLERS.map { | bill | bill.new(vm) } + @billers = BILLERS.map { | bill | bill.new(@vm) } @billers.select! { |bill| bill.check_biller } @timeline = Timeline.new vm, stime, etime From 4e750b2696196227ba7d2d2175950f976857e7d2 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:25:53 +0100 Subject: [PATCH 37/87] make_bill method --- service/showback.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/service/showback.rb b/service/showback.rb index d778087f..09130a9e 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -59,9 +59,21 @@ def initialize vm, stime, etime @timeline.init end + def make_bill + state, @bill = @timeline.state, [] + @timeline.timeline.each_cons(2) do | curr, con | + delta = con.ts - curr.ts + curr.mod state + bill_rec = {time: con.ts} + @billers.each do | biller | + bill_rec.merge! biller.bill(bill_rec, state, delta) + end + @bill << bill_rec + end + + @bill + end - def set_state state - @state = state end end From a8426d0847f9ad6aedb79d1f7a1b333fc1b03cb1 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:26:04 +0100 Subject: [PATCH 38/87] recept method --- service/showback.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/service/showback.rb b/service/showback.rb index 09130a9e..ea8cdf63 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -74,6 +74,11 @@ def make_bill @bill end + def receipt + @bill.map! do | el | + el.merge total: el.without(:time).values.sum + end + end end end From 82cab2f1f0deddfb46b971f4a34b5da3394aaeac Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:26:11 +0100 Subject: [PATCH 39/87] bill total method --- service/showback.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/service/showback.rb b/service/showback.rb index ea8cdf63..8309f221 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -79,6 +79,9 @@ def receipt el.merge total: el.without(:time).values.sum end end + + def total + @bill.inject(0) { |r, el| r += el[:total] } end end From 550a6b6cc7fbf9fe8a1047577807504bc72365ec Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:26:20 +0100 Subject: [PATCH 40/87] bill reader --- service/showback.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/showback.rb b/service/showback.rb index 8309f221..6d1d1bc4 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -43,7 +43,7 @@ def init # Class for billing through Timeline using different billers class Billing - attr_reader :timeline + attr_reader :timeline, :bill BILLERS = [ CapacityBiller From a3b833cc6cf1536427b0c07ed1a9719851eb6486 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 13:49:29 +0100 Subject: [PATCH 41/87] edge records --- service/showback.rb | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/service/showback.rb b/service/showback.rb index 6d1d1bc4..e5279977 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -28,7 +28,13 @@ def compile records.flatten! @timeline = records.sort_by { |rec| rec.sorter } - @timeline.select! { |rec| rec.sorter.between?(stime, etime)} + @timeline.select! { |rec| rec.sorter.between?(@stime, @etime)} + + init + init_rec = InitRecord.new @stime, @state + @timeline.unshift init_rec + @timeline << FinalRecord.new(@etime) + @compiled = true self end @@ -38,6 +44,26 @@ def init r.merge source.new(@vm.id).init_state(@stime) end end + + class InitRecord + def initialize time, state + @time, @state = time, state + end + def ts + @time + end + def mod st + st.merge! @state + end + end + class FinalRecord + def initialize time + @time = time + end + def ts + @time + end + end end # Class for billing through Timeline using different billers @@ -56,11 +82,10 @@ def initialize vm, stime, etime @timeline = Timeline.new vm, stime, etime @timeline.compile - @timeline.init end def make_bill - state, @bill = @timeline.state, [] + state, @bill = {}, [] @timeline.timeline.each_cons(2) do | curr, con | delta = con.ts - curr.ts curr.mod state From 5f09022eb81024af244244de32f8fc8468193621 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 14:21:03 +0100 Subject: [PATCH 42/87] showback test env base --- rake/tests/Rakefile | 17 ++++++++++ rake/tests/showback.rake | 67 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 rake/tests/Rakefile create mode 100644 rake/tests/showback.rake diff --git a/rake/tests/Rakefile b/rake/tests/Rakefile new file mode 100644 index 00000000..91ffec91 --- /dev/null +++ b/rake/tests/Rakefile @@ -0,0 +1,17 @@ +task :before_test do + require '/usr/lib/one/ione/ione_server.rb' +end + +def passed + puts "--- " + "Passed".green +end +def fail msg + puts msg.red + exit +end +def warn msg + puts msg.yellow +end + +load "rake/tests/showback.rake" + diff --git a/rake/tests/showback.rake b/rake/tests/showback.rake new file mode 100644 index 00000000..b38e8e8c --- /dev/null +++ b/rake/tests/showback.rake @@ -0,0 +1,67 @@ +desc "Test Capacity Showback" +task :cap_showback_test => :before_test do + puts "\n####################\n# Testing capacity #\n####################" + + $db[:settings].where(name: "CAPACITY_COST").update(body: "{\"CPU_COST\":\"0.5\",\"MEMORY_COST\":\"0.5\"}") + $db[:settings].where(name: "DISK_COSTS").update(body: "{\"HDD\":\"0.0\"}") + $db[:settings].where(name: "PUBLIC_IP_COST").update(body: "0.0") + + r = onblock(:vm, 3).calculate_showback 0, 1200 + if r[:TOTAL] == 600 then + passed + else + warn "#{0}, #{1200} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 1200 + if r[:TOTAL] == 300 then + passed + else + warn "#{300}, #{1200} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 500 + if r[:TOTAL] == 200 then + passed + else + warn "#{300}, #{500} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 1400 + if r[:TOTAL] == 500 then + passed + else + warn "#{300}, #{1400} => #{r[:TOTAL]}" + end +end + +desc "Test Disk Showback" +task :disk_showback_test => :before_test do + puts "\n####################\n# Testing capacity #\n####################" + + $db[:settings].where(name: "CAPACITY_COST").update(body: "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}") + $db[:settings].where(name: "DISK_COSTS").update(body: "{\"HDD\":\"1\"}") + $db[:settings].where(name: "PUBLIC_IP_COST").update(body: "0.0") + + r = onblock(:vm, 3).calculate_showback 0, 1200 + if r[:TOTAL] == 1200 then + passed + else + warn "#{0}, #{1200} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 1200 + if r[:TOTAL] == 900 then + passed + else + warn "#{300}, #{1200} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 500 + if r[:TOTAL] == 200 then + passed + else + warn "#{300}, #{500} => #{r[:TOTAL]}" + end + r = onblock(:vm, 3).calculate_showback 300, 1400 + if r[:TOTAL] == 1100 then + passed + else + warn "#{300}, #{1400} => #{r[:TOTAL]}" + end +end \ No newline at end of file From fde6b2af0a311f929994e393150c3333d505772b Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 14:21:21 +0100 Subject: [PATCH 43/87] disk biller v0 --- service/billers/disk.rb | 10 +++++++++- service/showback.rb | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/service/billers/disk.rb b/service/billers/disk.rb index 15a50d94..c1b13754 100644 --- a/service/billers/disk.rb +++ b/service/billers/disk.rb @@ -10,8 +10,16 @@ def check_biller rescue r end - return r > 0 + return false if r <= 0 + + @cost = r[@vm['/VM/USER_TEMPLATE/DRIVE']].to_f + return @cost > 0 rescue return false end + + def bill bill, state, delta + bill[:capacity] = delta * @cost + bill + end end \ No newline at end of file diff --git a/service/showback.rb b/service/showback.rb index e5279977..450658e1 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -72,7 +72,7 @@ class Billing attr_reader :timeline, :bill BILLERS = [ - CapacityBiller + CapacityBiller, DiskBiller ] def initialize vm, stime, etime From 1b1a67399376ef4160068bded450c1ef3e0a9080 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 14:27:14 +0100 Subject: [PATCH 44/87] disk biller fin --- rake/tests/showback.rake | 7 +++++-- service/billers/disk.rb | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/rake/tests/showback.rake b/rake/tests/showback.rake index b38e8e8c..de88acdc 100644 --- a/rake/tests/showback.rake +++ b/rake/tests/showback.rake @@ -34,7 +34,7 @@ end desc "Test Disk Showback" task :disk_showback_test => :before_test do - puts "\n####################\n# Testing capacity #\n####################" + puts "\n################\n# Testing Disk #\n################" $db[:settings].where(name: "CAPACITY_COST").update(body: "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}") $db[:settings].where(name: "DISK_COSTS").update(body: "{\"HDD\":\"1\"}") @@ -64,4 +64,7 @@ task :disk_showback_test => :before_test do else warn "#{300}, #{1400} => #{r[:TOTAL]}" end -end \ No newline at end of file +end + +desc "Showback Tests" +task :showback_test => [:cap_showback_test, :disk_showback_test] do; end \ No newline at end of file diff --git a/service/billers/disk.rb b/service/billers/disk.rb index c1b13754..abaeca16 100644 --- a/service/billers/disk.rb +++ b/service/billers/disk.rb @@ -1,7 +1,7 @@ class DiskBiller < Biller # Checking if Capacity costs are given, otherwise there is no point to calculate it def check_biller - @costs = JSON.parse(costs['DISK_COST']) + @costs = JSON.parse(costs['DISK_COSTS']) return false if @costs.nil? r = @@ -12,7 +12,7 @@ def check_biller end return false if r <= 0 - @cost = r[@vm['/VM/USER_TEMPLATE/DRIVE']].to_f + @cost = @costs[@vm['/VM/USER_TEMPLATE/DRIVE']].to_f return @cost > 0 rescue return false From 8ae50bdf821c974c12681192520e44d5a24d1795 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 15 Dec 2020 14:36:05 +0100 Subject: [PATCH 45/87] ptch --- rake/tests/showback.rake | 2 -- 1 file changed, 2 deletions(-) diff --git a/rake/tests/showback.rake b/rake/tests/showback.rake index de88acdc..8e1e0168 100644 --- a/rake/tests/showback.rake +++ b/rake/tests/showback.rake @@ -4,7 +4,6 @@ task :cap_showback_test => :before_test do $db[:settings].where(name: "CAPACITY_COST").update(body: "{\"CPU_COST\":\"0.5\",\"MEMORY_COST\":\"0.5\"}") $db[:settings].where(name: "DISK_COSTS").update(body: "{\"HDD\":\"0.0\"}") - $db[:settings].where(name: "PUBLIC_IP_COST").update(body: "0.0") r = onblock(:vm, 3).calculate_showback 0, 1200 if r[:TOTAL] == 600 then @@ -38,7 +37,6 @@ task :disk_showback_test => :before_test do $db[:settings].where(name: "CAPACITY_COST").update(body: "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}") $db[:settings].where(name: "DISK_COSTS").update(body: "{\"HDD\":\"1\"}") - $db[:settings].where(name: "PUBLIC_IP_COST").update(body: "0.0") r = onblock(:vm, 3).calculate_showback 0, 1200 if r[:TOTAL] == 1200 then From 059fae5b59c0f728fd28684fade41f276cd9f45e Mon Sep 17 00:00:00 2001 From: Mik Date: Thu, 17 Dec 2020 14:09:55 +0100 Subject: [PATCH 46/87] int -> float --- service/billers/disk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/billers/disk.rb b/service/billers/disk.rb index abaeca16..7a336771 100644 --- a/service/billers/disk.rb +++ b/service/billers/disk.rb @@ -6,7 +6,7 @@ def check_biller r = @costs.values.inject(0) do | r, c | - r += c.to_i + r += c.to_f rescue r end From 88cfdacb694f15f2d78f0d890e1357a7ef0cdd5e Mon Sep 17 00:00:00 2001 From: Mik Date: Thu, 17 Dec 2020 14:10:00 +0100 Subject: [PATCH 47/87] traffic base --- service/billers/traffic.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 service/billers/traffic.rb diff --git a/service/billers/traffic.rb b/service/billers/traffic.rb new file mode 100644 index 00000000..6ef5b400 --- /dev/null +++ b/service/billers/traffic.rb @@ -0,0 +1,11 @@ +class TrafficBiller < Biller + def check_biller + @cost = JSON.parse(costs['TRAFFIC_COST']) + return false if @cost.nil? + + @cost = @cost.to_f + return @cost > 0 + rescue + return false + end +end \ No newline at end of file From 94c420df592946f2ddcf00a46ecb05f6a8c7b355 Mon Sep 17 00:00:00 2001 From: Mik Date: Thu, 17 Dec 2020 14:12:54 +0100 Subject: [PATCH 48/87] snapshots biller beta --- service/billers/snapshots.rb | 16 ++++++++++++++++ service/showback.rb | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 service/billers/snapshots.rb diff --git a/service/billers/snapshots.rb b/service/billers/snapshots.rb new file mode 100644 index 00000000..f8924bb1 --- /dev/null +++ b/service/billers/snapshots.rb @@ -0,0 +1,16 @@ +class SnapshotsBiller < Biller + def check_biller + @cost = JSON.parse(costs['SNAPSHOT_COST']) + return false if @cost.nil? + + @cost = @cost.to_f + return @cost > 0 + rescue + return false + end + + def bill bill, state, delta + bill[:snapshots] = delta * @cost * state[:snaps] + bill + end +end \ No newline at end of file diff --git a/service/showback.rb b/service/showback.rb index 450658e1..b833f951 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -72,7 +72,7 @@ class Billing attr_reader :timeline, :bill BILLERS = [ - CapacityBiller, DiskBiller + CapacityBiller, DiskBiller, SnapshotsBiller ] def initialize vm, stime, etime From 7e965be74a6002b53464612ec945018f7355725c Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 23 Dec 2020 13:03:48 +0200 Subject: [PATCH 49/87] traffic_records table --- service/objects/records.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/service/objects/records.rb b/service/objects/records.rb index a43a8f6a..6ef1765b 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -21,6 +21,18 @@ puts "Table :snapshot_records already exists, skipping" end +begin + $db.create_table :traffic_records do + primary_key :key + foreign_key :vm, :vm_pool, null: false + Integer :ts, null: false + Integer :val, null: false + String :type, null: false + end +rescue + puts "Table :traffic_records already exists, skipping" +end + # History Record Model class # @see https://github.com/ione-cloud/ione-sunstone/blob/55a9efd68681829624809b4895a49d750d6e6c34/ione/server/service/objects/records.rb#L1-L10 History Model Defintion class Record < Sequel::Model(:records) From 21ce3fa18a4229627a3bf118df5264d94df82b8d Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 23 Dec 2020 15:21:23 +0200 Subject: [PATCH 50/87] traffic recorder cron script --- scripts/traffic-recorder/main.rb | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 scripts/traffic-recorder/main.rb diff --git a/scripts/traffic-recorder/main.rb b/scripts/traffic-recorder/main.rb new file mode 100644 index 00000000..906d67e4 --- /dev/null +++ b/scripts/traffic-recorder/main.rb @@ -0,0 +1,35 @@ +begin + LOG 'Traffic Recorder has been initialized', 'TrafficRecorder' + loop do + begin + vm_pool = VirtualMachinePool.new($client) + vm_pool.info_all + inserts_total = 0 + + vm_pool.each do | vm | + mon = vm.monitoring(['NETTX', 'NETRX']) + inserts = 0 + for key, data in mon do + last = TrafficRecord.where(type: key, vm: vm.id).order(Sequel.asc(:ts)).last + last = { ts: 0 } if last.nil? + for rec in data do + if rec.first.to_i > last[:ts] then + TrafficRecord.insert vm: vm.id, ts: rec.first.to_i, val: rec.last, type: key + inserts += 1 + end + end + end + LOG "VM #{vm.id}: Inserted #{inserts} new traffic records", "TrafficRecorder" + inserts_total += inserts + end + + LOG "TrafficRecorder inserted totally #{inserts_total} new traffic records", 'TrafficRecorder' + sleep($ione_conf['TrafficRecorder']['check-period']) + rescue => e + LOG "TrafficRecorder Error, code: #{e.message}\TrafficRecorder is down now", 'TrafficRecorder' + sleep(30) + end + end +rescue + LOG "TrafficRecorder fatal error, service is crashed", 'TrafficRecorderThread' +end \ No newline at end of file From 5c52809fc1e0cc86392bdd447f1c6f2cd9103a80 Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 23 Dec 2020 15:21:40 +0200 Subject: [PATCH 51/87] traffic recorder script logs --- service/log.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/service/log.rb b/service/log.rb index 5becfa5b..7303f72d 100644 --- a/service/log.rb +++ b/service/log.rb @@ -31,6 +31,8 @@ def LOG(msg, method = "none", _time = true) destination = "#{LOG_ROOT}/debug.log" when "SnapController" destination = "#{LOG_ROOT}/snapshot.log" + when "TrafficRecorder" + destination = "#{LOG_ROOT}/traffic_recorder.log" else destination = "#{LOG_ROOT}/ione.log" end @@ -49,6 +51,7 @@ def LOG(msg, method = "none", _time = true) def LOG_COLOR(msg, method = caller_locations(1,1)[0].label.dup, color = 'red', font = 'bold') destination = "#{LOG_ROOT}/ione.log" destination = "#{LOG_ROOT}/snapshot.log" if method == "SnapController" + destination = "#{LOG_ROOT}/traffic_recorder.log" if method == "TrafficRecorder" msg = msg.to_s.send(color).send(font) msg = "[ #{Time.now.ctime} ] " + msg method.slice!('block in '.dup) @@ -83,6 +86,8 @@ def LOG_AUTO(msg, method = caller_locations(1,1)[0].label, _time = true) destination = "#{LOG_ROOT}/debug.log" when "SnapController" destination = "#{LOG_ROOT}/snapshot.log" + when "TrafficRecorder" + destination = "#{LOG_ROOT}/traffic_recorder.log" else destination = "#{LOG_ROOT}/ione.log" end From 7b0c8bf045bf7ee97f628754acb3650b8185cdd0 Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 23 Dec 2020 15:21:45 +0200 Subject: [PATCH 52/87] traffic recorder conf --- sys/ione.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sys/ione.conf b/sys/ione.conf index e36cf249..18fc75ba 100644 --- a/sys/ione.conf +++ b/sys/ione.conf @@ -37,6 +37,9 @@ vCenter: SnapshotController: check-period: 3600 # Snapshots check frequency in seconds +TrafficRecorder: + check-period: 3600 # Traffic monitoring check frequency in seconds + DB: # This values are available in /etc/one/oned.conf user: user pass: secret From 60fdc837327dc36498d966801728cbfb0e0ed5de Mon Sep 17 00:00:00 2001 From: Mik Date: Wed, 23 Dec 2020 15:21:58 +0200 Subject: [PATCH 53/87] traffic recorder table alter --- service/objects/records.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/service/objects/records.rb b/service/objects/records.rb index 6ef1765b..51bed28d 100644 --- a/service/objects/records.rb +++ b/service/objects/records.rb @@ -26,7 +26,7 @@ primary_key :key foreign_key :vm, :vm_pool, null: false Integer :ts, null: false - Integer :val, null: false + String :val, null: false String :type, null: false end rescue @@ -86,6 +86,12 @@ def sortable end end +class TrafficRecord < Sequel::Model(:traffic_records) + def sorter + ts + end +end + # Source of History records class class RecordsSource attr_reader :id From dcdd283d75bfacbb26ab0e961ae75190723b4b86 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 25 Dec 2020 13:47:54 +0200 Subject: [PATCH 54/87] move records and detach traffic records and recordssource --- service/records.rb | 40 +++++++++++++++++ service/{objects => records}/records.rb | 57 ------------------------- service/records/traffic_records.rb | 39 +++++++++++++++++ 3 files changed, 79 insertions(+), 57 deletions(-) create mode 100644 service/records.rb rename service/{objects => records}/records.rb (70%) create mode 100644 service/records/traffic_records.rb diff --git a/service/records.rb b/service/records.rb new file mode 100644 index 00000000..7e6f7df9 --- /dev/null +++ b/service/records.rb @@ -0,0 +1,40 @@ +# Source of History records class +class RecordsSource + attr_reader :id + + # VMID field key + def key + :id + end + + # @param [Fixnum] id - VM ID + def initialize cls, id + @id = id + @records = cls.where(Hash[key, @id]) + end + + def records + @records.all + end + + # Find records for given time period + def find stime, etime + @records.where(time: stime..etime) + end + + def init_state stime + {} + end + + # Filter records needed for Showback Timeline + def self.tl_filter records + records + end + + # Check if source should be used with given VM + def self.check_source vm + true + end +end + +Dir["#{ROOT}/service/records/*.rb"].each {|file| require file } diff --git a/service/objects/records.rb b/service/records/records.rb similarity index 70% rename from service/objects/records.rb rename to service/records/records.rb index 51bed28d..57ead328 100644 --- a/service/objects/records.rb +++ b/service/records/records.rb @@ -21,18 +21,6 @@ puts "Table :snapshot_records already exists, skipping" end -begin - $db.create_table :traffic_records do - primary_key :key - foreign_key :vm, :vm_pool, null: false - Integer :ts, null: false - String :val, null: false - String :type, null: false - end -rescue - puts "Table :traffic_records already exists, skipping" -end - # History Record Model class # @see https://github.com/ione-cloud/ione-sunstone/blob/55a9efd68681829624809b4895a49d750d6e6c34/ione/server/service/objects/records.rb#L1-L10 History Model Defintion class Record < Sequel::Model(:records) @@ -86,51 +74,6 @@ def sortable end end -class TrafficRecord < Sequel::Model(:traffic_records) - def sorter - ts - end -end - -# Source of History records class -class RecordsSource - attr_reader :id - - # VMID field key - def key - :id - end - - # @param [Fixnum] id - VM ID - def initialize cls, id - @id = id - @records = cls.where(Hash[key, @id]) - end - - def records - @records.all - end - - # Find records for given time period - def find stime, etime - @records.where(time: stime..etime) - end - - def init_state stime - {} - end - - # Filter records needed for Showback Timeline - def self.tl_filter records - records - end - - # Check if source should be used with given VM - def self.check_source vm - true - end -end - class OpenNebula::Records < RecordsSource def initialize id super(Record, id) diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb new file mode 100644 index 00000000..c2429273 --- /dev/null +++ b/service/records/traffic_records.rb @@ -0,0 +1,39 @@ +begin + $db.create_table :traffic_records do + primary_key :key + foreign_key :vm, :vm_pool, null: false + Integer :ts, null: false + String :rx, null: false + String :tx, null: false + end +rescue + puts "Table :traffic_records already exists, skipping" +end + +class TrafficRecord < Sequel::Model(:traffic_records) + def sortable + self + end + def sorter + ts + end + def mod st + st[:traffic] = 0 + end +end + +class OpenNebula::TrafficRecords < RecordsSource + + def key + :vm + end + + def initialize id + super(TrafficRecord, id) + end + + def find stime, etime + @records.where{ ts < stime }.order(Sequel.asc :ts).limit(1) + end + +end \ No newline at end of file From 96a0945742b3440baf5bdc1b23d1bb8e94082451 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 25 Dec 2020 13:48:10 +0200 Subject: [PATCH 55/87] defuq --- service/showback.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/service/showback.rb b/service/showback.rb index b833f951..436b8a94 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -108,8 +108,4 @@ def receipt def total @bill.inject(0) { |r, el| r += el[:total] } end -end - - self - end -end +end \ No newline at end of file From c1026a39308bd1aede34f6334344cc81b5d6312d Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 25 Dec 2020 13:48:17 +0200 Subject: [PATCH 56/87] use records --- service/showback.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/service/showback.rb b/service/showback.rb index 436b8a94..dee19d1a 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -1,3 +1,4 @@ +require "#{ROOT}/service/records.rb" require "#{ROOT}/service/biller.rb" # Class for compiling all history records from all records sources into one readable timeline From ccc56d094f3ff694e6233b902981e8ff7723d471 Mon Sep 17 00:00:00 2001 From: Mik Date: Fri, 25 Dec 2020 13:48:32 +0200 Subject: [PATCH 57/87] following updates in traffic records schema --- scripts/traffic-recorder/main.rb | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/scripts/traffic-recorder/main.rb b/scripts/traffic-recorder/main.rb index 906d67e4..17629242 100644 --- a/scripts/traffic-recorder/main.rb +++ b/scripts/traffic-recorder/main.rb @@ -7,16 +7,26 @@ inserts_total = 0 vm_pool.each do | vm | - mon = vm.monitoring(['NETTX', 'NETRX']) + mon_raw = vm.monitoring(['NETTX', 'NETRX']) + mon = {} + mon_raw['NETTX'].each do | el | + mon[el.first] = {} + mon[el.first][:tx] = el.last + end + mon_raw['NETRX'].each do | el | + mon[el.first][:rx] = el.last + end + inserts = 0 - for key, data in mon do - last = TrafficRecord.where(type: key, vm: vm.id).order(Sequel.asc(:ts)).last - last = { ts: 0 } if last.nil? - for rec in data do - if rec.first.to_i > last[:ts] then - TrafficRecord.insert vm: vm.id, ts: rec.first.to_i, val: rec.last, type: key - inserts += 1 - end + last = TrafficRecord.where(vm: vm.id).order(Sequel.asc(:ts)).last + last = { ts: 0 } if last.nil? + + for ts, data in mon do + ts = ts.to_i + if ts > last[:ts] then + args = data.merge(vm: vm.id, ts: ts) + TrafficRecord.insert **args + inserts += 1 end end LOG "VM #{vm.id}: Inserted #{inserts} new traffic records", "TrafficRecorder" From a3efc2809e875628cf011df2d85e6987fada7aa8 Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 28 Dec 2020 10:53:33 +0200 Subject: [PATCH 58/87] sync traffic records from monitoring on initialize --- scripts/traffic-recorder/main.rb | 23 +------------------ service/records/traffic_records.rb | 36 +++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/scripts/traffic-recorder/main.rb b/scripts/traffic-recorder/main.rb index 17629242..8761f57c 100644 --- a/scripts/traffic-recorder/main.rb +++ b/scripts/traffic-recorder/main.rb @@ -7,28 +7,7 @@ inserts_total = 0 vm_pool.each do | vm | - mon_raw = vm.monitoring(['NETTX', 'NETRX']) - mon = {} - mon_raw['NETTX'].each do | el | - mon[el.first] = {} - mon[el.first][:tx] = el.last - end - mon_raw['NETRX'].each do | el | - mon[el.first][:rx] = el.last - end - - inserts = 0 - last = TrafficRecord.where(vm: vm.id).order(Sequel.asc(:ts)).last - last = { ts: 0 } if last.nil? - - for ts, data in mon do - ts = ts.to_i - if ts > last[:ts] then - args = data.merge(vm: vm.id, ts: ts) - TrafficRecord.insert **args - inserts += 1 - end - end + inserts = TrafficRecords.new(vm.id, true).sync vm LOG "VM #{vm.id}: Inserted #{inserts} new traffic records", "TrafficRecorder" inserts_total += inserts end diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index c2429273..e54e46f8 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -28,8 +28,42 @@ def key :vm end - def initialize id + def initialize id, nosync = false super(TrafficRecord, id) + sync unless nosync + end + + def sync vm = nil + inserts = 0 + + if vm.nil? then + vm = onblock :vm, @id + vm.info! + end + + mon_raw = vm.monitoring(['NETTX', 'NETRX']) + mon = {} + mon_raw['NETTX'].each do | el | + mon[el.first] = {} + mon[el.first][:tx] = el.last + end + mon_raw['NETRX'].each do | el | + mon[el.first][:rx] = el.last + end + + last = TrafficRecord.where(vm: vm.id).order(Sequel.asc(:ts)).last + last = { ts: 0 } if last.nil? + + for ts, data in mon do + ts = ts.to_i + if ts > last[:ts] then + args = data.merge(vm: vm.id, ts: ts) + TrafficRecord.insert **args + inserts += 1 + end + end + + inserts end def find stime, etime From 8f2bde013396fdeb92dcb1982ab803f7a1a95736 Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 28 Dec 2020 11:35:44 +0200 Subject: [PATCH 59/87] calculate showback method test --- service/objects/vm.rb | 212 ++---------------------------------------- 1 file changed, 6 insertions(+), 206 deletions(-) diff --git a/service/objects/vm.rb b/service/objects/vm.rb index 75730178..01be2474 100644 --- a/service/objects/vm.rb +++ b/service/objects/vm.rb @@ -20,12 +20,6 @@ class OpenNebula::VirtualMachine undeploy-hard snapshot-create ) - # List of possible billing periods - # @note Read it in source by pressing button View source down - BILLING_PERIODS = [ - "0", # Pay-as-you-Go -- default - /\d/, # Number of days - ] # Generates template for OpenNebula scheduler record def generate_schedule_str(id, action, time) "\nSCHED_ACTION=[\n" + @@ -357,209 +351,15 @@ def calculate_showback stime_req, etime_req, group_by_day = false stime = self['/VM/STIME'].to_i if self['/VM/STIME'].to_i > stime etime = self['/VM/ETIME'].to_i if self['/VM/ETIME'].to_i < etime && self['/VM/ETIME'].to_i != 0 - requested_time = (etime - stime) / 3600.0 - - ### Quick response for HOLD and PENDING vms ### - return { - "id" => id, - "name" => name, - "work_time" => 0, - "time_period_requested" => etime_req - stime_req, - "time_period_corrected" => etime - stime, - "CPU" => 0, - "MEMORY" => 0, - "DISK" => 0, - "DISK_TYPE" => self['/VM/USER_TEMPLATE/DRIVE'], - "EXCEPTION" => "State #{state == 0 ? "HOLD" : "PENDING"}", - "TOTAL" => 0 - } if state == 0 || state == 1 - - records = OpenNebula::Records.new(id).records - - if self['//BILLING_PERIOD'] == 'month' then - first = records.select{|r| r[:state] != 'pnd'}.sort_by{|r| r[:time]}.first[:time] - - unless group_by_day then - stime = Time.at(stime).to_datetime - first = Time.at(first).to_datetime - etime = Time.at(etime).to_datetime - current, periods = stime > first ? stime : first, 0 - - while current <= etime do - periods += 1 - current = current >> 1 - end - return {"id" => id, "name" => name, "work_time" => etime.to_time.to_i - first.to_time.to_i, "EXCEPTION" => "Billed by calendar month", "TOTAL" => (periods * self['//PRICE'].to_f).round(2)} - end - elsif self['//BILLING_PERIOD'].to_i != 0 then - - first = records.select{|r| r[:state] != 'pnd'}.sort_by{|r| r[:time]}.first[:time] - delta = self['//BILLING_PERIOD'].to_i * 86400 - - unless group_by_day then - - periods = stime > first ? 0 : 1 - periods += (etime - first) / delta - - return {"id" => id, "name" => name, "work_time" => etime - first, "EXCEPTION" => "Billed per #{self['//BILLING_PERIOD'].to_i} days", "TOTAL" => (periods * self['//PRICE'].to_f).round(2)} - else - showback = [] + billing = Billing.new self, stime, etime + billing.make_bill + billing.receipt - diff = (Time.at(etime).to_date - Time.at(first).to_date).to_i + 1 - diff.times do | day | - showback << { - 'date' => Time.at(first + day * 86400).to_a[3..5].join('/'), - 'TOTAL' => ( - day % self['//BILLING_PERIOD'].to_i == 0 ? self['//PRICE'].to_f : 0.0 ) - } - end - return { - "id" => id, "name" => name, "work_time" => first - stime, - "EXCEPTION" => "Billed per #{self['//BILLING_PERIOD'].to_i} days", - "showback" => showback, - "TOTAL" => showback.inject(0){|total, record| total + record['TOTAL']} - } - end - end - - - - ### Generating Timeline ### - timeline = [] - records.each_with_index do | record, i | - timeline << { - 'stime' => record[:time], - 'etime' => i + 1 != records.size ? records[i + 1][:time] - 1 : etime, - 'state' => record[:state] - } - end - - timeline.delete_if { |r| (r['etime'] < stime) || (r['stime'] > etime) } - raise OpenNebula::Records::NoRecordsError if timeline.empty? - timeline[0]['stime'] = stime if timeline[0]['stime'] < stime - timeline[-1]['etime'] = etime if timeline.last['etime'] > etime - - - ### Calculating Capacity ### - cpu = self['/VM/TEMPLATE/VCPU'].to_f * self['/VM/TEMPLATE/CPU'].to_f - memory = self['/VM/TEMPLATE/MEMORY'].to_f / 1024 - disk = self['/VM/TEMPLATE/DISK/SIZE'].to_f / 1024 - - ### Calculating Showback ### - cpu_cost = cpu * self['/VM/TEMPLATE/CPU_COST'].to_f - memory_cost = memory * self['/VM/TEMPLATE/MEMORY_COST'].to_f - disk_cost = disk * self['/VM/TEMPLATE/DISK_COST'].to_f - - unless group_by_day then - ### Calculating Work Time ### - work_time = 0 - timeline.each do | record | - next unless record['state'] == 'on' - work_time += record['etime'] - record['stime'] - end - work_time = work_time / 3600.0 - - - - ### Calculating Showback ### - cpu_cost *= work_time - memory_cost *= work_time - disk_cost *= requested_time - - return { - "id" => id, - "name" => name, - "work_time" => work_time, - "time_period_requested" => etime_req - stime_req, - "time_period_corrected" => etime - stime, - "CPU" => cpu_cost, - "MEMORY" => memory_cost, - "DISK" => disk_cost, - "DISK_TYPE" => self['/VM/USER_TEMPLATE/DRIVE'], - "TOTAL" => cpu_cost + memory_cost + disk_cost - } - else - timeline.clone.each_with_index do | r, i | - diff = (Time.at(r['etime']).to_date - Time.at(r['stime']).to_date).to_i - if diff >= 1 then - result, border = [], Time.at(r['stime']).to_a - border[0..2] = 59, 59, 23 - border = Time.local(*border).to_i - - result << { 'stime' => r['stime'], 'etime' => border, 'state' => r['state'], 'date' => Time.at(r['stime']).to_a[3..5].join('/') } - - (diff).times do | day | - result << { 'stime' => border += 1, 'date' => Time.at(border).to_a[3..5].join('/'), 'etime' => border += 86399, 'state' => r['state'] } - end - - result[diff]['etime'] = r['etime'] - - timeline[i] = result - else - timeline[i]['date'] = Time.at(r['stime']).to_a[3..5].join('/') - end - end - timeline.flatten! - - timeline.map! do | r | - { - "date" => r['date'], - "requested_time" => (r['etime'] - r['stime']), - 'state' => r['state'] - } - end - timeline = timeline.group_by { | r | r['date'] } - timeline = timeline.map do | date, date_records | - result = date_records.inject({ - 'date' => date, - 'work_time' => 0, - 'requested_time' => 0, - 'CPU' => 0, - 'MEMORY' => 0, - 'DISK' => 0, - 'TOTAL' => 0 - }) do | showback, record | - requested_time = record['requested_time'] / 3600.0 - work_time = record['state'] == 'on' ? requested_time : 0 - showback['work_time'] += work_time - showback['requested_time'] += record['requested_time'] - showback['CPU'] += cpu_cost * work_time - showback['MEMORY'] += memory_cost * work_time - showback['DISK'] += disk_cost * requested_time if record['state'] != 'pnd' - - showback - end - - result['TOTAL'] += (result['CPU'] + result['MEMORY'] + result['DISK']) - result - end - - return { - "id" => id, - "name" => name, - "work_time" => timeline.inject(0){|total, record| total + record['work_time']}, - "requested_time" => timeline.inject(0){|total, record| total + record['requested_time']}, - "time_period_requested" => etime_req - stime_req, - "time_period_corrected" => etime - stime, - "showback" => timeline, - "DISK_TYPE" => self['/VM/USER_TEMPLATE/DRIVE'], - "TOTAL" => timeline.inject(0){|total, record| total + record['TOTAL']} - } - end - rescue OpenNebula::Records::NoRecordsError return { - "id" => id, - "name" => name, - "work_time" => 0, - "time_period_requested" => etime_req - stime_req, - "time_period_corrected" => etime - stime, - "CPU" => 0, - "MEMORY" => 0, - "DISK" => 0, - "DISK_TYPE" => 'no_type', - "EXCEPTION" => "No Records", - "TOTAL" => 0 + id: id, name: name, + showback: billing.bill, + TOTAL: billing.total } end # Returns important data in JSON format From d7cd70f82aabccb4099645b49e48693534dcc8ba Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 28 Dec 2020 11:49:34 +0200 Subject: [PATCH 60/87] wrong key fix --- service/billers/disk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/billers/disk.rb b/service/billers/disk.rb index 7a336771..008839d4 100644 --- a/service/billers/disk.rb +++ b/service/billers/disk.rb @@ -19,7 +19,7 @@ def check_biller end def bill bill, state, delta - bill[:capacity] = delta * @cost + bill[:disk] = delta * @cost bill end end \ No newline at end of file From 42576adba496c22d3f179c20b0c4f5f164f45b1a Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 28 Dec 2020 11:49:51 +0200 Subject: [PATCH 61/87] bill if state on or not PAYG --- service/billers/capacity.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/billers/capacity.rb b/service/billers/capacity.rb index 0195d98e..78801d6c 100644 --- a/service/billers/capacity.rb +++ b/service/billers/capacity.rb @@ -16,7 +16,7 @@ def check_biller end def bill bill, state, delta - if state[:state] == 'on' then + if state[:state] == 'on' || billing_period != 'PAYG' then bill[:capacity] = delta * @cost end bill From 2e3b81bdb3ec525ee73bb76f03714a6ef44932cb Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 28 Dec 2020 11:50:04 +0200 Subject: [PATCH 62/87] billing_period method for Biller --- service/biller.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service/biller.rb b/service/biller.rb index c0d6bf75..9f294666 100644 --- a/service/biller.rb +++ b/service/biller.rb @@ -14,6 +14,10 @@ def check_biller true end + def billing_period + @vm['//BILLING_PERIOD'] + end + def bill bill, state, delta 0 end From aea387cf06c662f46b5ef5ff9bd3806d13adae7f Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 28 Dec 2020 11:50:20 +0200 Subject: [PATCH 63/87] IONe Showback notes --- docs/Showback.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/Showback.md diff --git a/docs/Showback.md b/docs/Showback.md new file mode 100644 index 00000000..b016528f --- /dev/null +++ b/docs/Showback.md @@ -0,0 +1,8 @@ +# Notes about configuring VM Showback + +## BILLING_PERIOD + +Each and every VM should have BILLING_PERIOD defined in template. +Possible values are: + - PAYG - VM will be billed per second + - PRE_<% n_days %> - Pre-Paid per N days, e.g. `PRE_30`, then full Capacity and Disk cost will be charged on deploy, and then every 30 days. \ No newline at end of file From 4fcf6278e69c1e1338b8aae6736a42d9ef2e8d2a Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 28 Dec 2020 12:10:31 +0200 Subject: [PATCH 64/87] h8 brckts --- service/objects/vm.rb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/service/objects/vm.rb b/service/objects/vm.rb index 01be2474..5aa35f98 100644 --- a/service/objects/vm.rb +++ b/service/objects/vm.rb @@ -21,7 +21,7 @@ class OpenNebula::VirtualMachine snapshot-create ) # Generates template for OpenNebula scheduler record - def generate_schedule_str(id, action, time) + def generate_schedule_str id, action, time "\nSCHED_ACTION=[\n" + " ACTION=\"#{action}\",\n" + " ID=\"#{id}\",\n" + @@ -37,7 +37,7 @@ def schedule_actions # @param [Integer] time - Time when action schould be perfomed in secs # @param [String] periodic - Not working now # @return true - def schedule(action, time, periodic = nil) + def schedule action, time, periodic = nil return 'Unsupported action' if !SCHEDULABLE_ACTIONS.include? action self.info! id = @@ -62,7 +62,7 @@ def schedule(action, time, periodic = nil) end # Unschedules given action by ID # @note Not working, if action is already initialized - def unschedule(id) + def unschedule id self.info! schedule_data, object = self.to_hash['VM']['USER_TEMPLATE']['SCHED_ACTION'], nil @@ -91,7 +91,7 @@ def scheduler # @param [Integer] s - VM state to wait for # @param [Integer] lcm_s - VM LCM state to wait for # @return [Boolean] - def wait_for_state(s = 3, lcm_s = 3) + def wait_for_state s = 3, lcm_s = 3 i = 0 until state!() == s && lcm_state!() == lcm_s do return false if i >= 3600 @@ -119,7 +119,7 @@ def wait_for_state(s = 3, lcm_s = 3) # => 'Reconfigure Success' -- Task finished with success code, all specs are equal to given # => 'Reconfigure Unsuccessed' -- Some of specs didn't changed # => 'Reconfigure Error:{error message}' -- Exception has been generated while proceed, check your configuration - def setResourcesAllocationLimits(spec) + def setResourcesAllocationLimits spec LOG_DEBUG spec.debug_out return 'Unsupported query' if IONe.new($client, $db).get_vm_data(self.id)['IMPORTED'] == 'YES' @@ -163,7 +163,7 @@ def setResourcesAllocationLimits(spec) return nil end # Checks if vm is on given vCenter Datastore - def is_at_ds?(ds_name) + def is_at_ds? ds_name host = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)) datacenter = get_vcenter_dc(host) begin @@ -199,7 +199,7 @@ def get_vms_vcenter_ds # @option spec [String] :name VM name on vCenter node # @return [Boolean | String] # @note Method returns true if resize action ended correct, false if VM not support hot reconfiguring - def hot_resize(spec = {:name => nil}) + def hot_resize spec = {:name => nil} return false if !self.hotAddEnabled? begin host = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)) @@ -221,7 +221,7 @@ def hot_resize(spec = {:name => nil}) # @note For correct work of this method, you must keep actual vCenter Password at VCENTER_PASSWORD_ACTUAL attribute in OpenNebula # @note Method searches VM by it's default name: one-(id)-(name), if target vm got another name, you should provide it # @return [Hash | String] Returns limits Hash if success or exception message if fails - def hotAddEnabled?(name = nil) + def hotAddEnabled? name = nil begin host = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)) datacenter = get_vcenter_dc(host) @@ -240,7 +240,7 @@ def hotAddEnabled?(name = nil) # @option spec [Boolean] :ram # @option spec [String] :name VM name on vCenter node # @return [true | String] - def hotResourcesControlConf(spec = {:cpu => true, :ram => true, :name => nil}) + def hotResourcesControlConf spec = {:cpu => true, :ram => true, :name => nil} begin host, name = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)), spec[:name] datacenter = get_vcenter_dc(host) @@ -275,7 +275,7 @@ def hotResourcesControlConf(spec = {:cpu => true, :ram => true, :name => nil}) # @note For correct work of this method, you must keep actual vCenter Password at VCENTER_PASSWORD_ACTUAL attribute in OpenNebula # @note Method searches VM by it's default name: one-(id)-(name), if target vm got another name, you should provide it # @return [Hash | String] Returns limits Hash if success or exception message if fails - def getResourcesAllocationLimits(name = nil) + def getResourcesAllocationLimits name = nil begin host = onblock(:h, IONe.new($client, $db).get_vm_host(self.id)) datacenter = get_vcenter_dc(host) @@ -293,7 +293,7 @@ def getResourcesAllocationLimits(name = nil) # Returns owner user ID # @param [Boolean] info - method doesn't get object full info one more time -- usefull if collecting data from pool # @return [Integer] - def uid(info = true) + def uid info = true self.info! if info self['UID'] end @@ -301,7 +301,7 @@ def uid(info = true) # @param [Boolean] info - method doesn't get object full info one more time -- usefull if collecting data from pool # @param [Boolean] from_pool - levels differenct between object and object received from pool.each | object | # @return [String] - def uname(info = true, from_pool = false) + def uname info = true, from_pool = false self.info! if info return @xml[0].children[3].text.to_i unless from_pool @xml.children[3].text From 9420de55f0ddf5efd2059b4c4c9366aadff94e84 Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 28 Dec 2020 12:24:02 +0200 Subject: [PATCH 65/87] billing period PRE PAID --- service/objects/vm.rb | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/service/objects/vm.rb b/service/objects/vm.rb index 5aa35f98..0824e525 100644 --- a/service/objects/vm.rb +++ b/service/objects/vm.rb @@ -351,16 +351,41 @@ def calculate_showback stime_req, etime_req, group_by_day = false stime = self['/VM/STIME'].to_i if self['/VM/STIME'].to_i > stime etime = self['/VM/ETIME'].to_i if self['/VM/ETIME'].to_i < etime && self['/VM/ETIME'].to_i != 0 + bp = self['//BILLING_PERIOD'] - billing = Billing.new self, stime, etime - billing.make_bill - billing.receipt + if bp.nil? || bp == 'PAYG' then + billing = Billing.new self, stime, etime + billing.make_bill + billing.receipt + + return { + id: id, name: name, + showback: billing.bill, + TOTAL: billing.total + } + elsif bp.include? 'PRE' then + curr = self['/VM/STIME'].to_i + delta = bp.split('_')[1].to_i * 86400 - return { - id: id, name: name, - showback: billing.bill, - TOTAL: billing.total - } + s, e = self['/VM/STIME'].to_i, self['/VM/ETIME'].to_i + total = 0 + + while curr < etime do + if (stime..etime).include? curr then + b = Billing.new self, curr, curr + delta + b.make_bill + b.receipt + + total += b.total + end + curr += delta + end + + return { + id: id, name: name, + TOTAL: total + } + end end # Returns important data in JSON format # @param [IONe] ione - IONe object for calling its methods From bb2c64bcdb0883351a1dceb64602d8d531485c96 Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 4 Jan 2021 15:12:43 +0200 Subject: [PATCH 66/87] ah? --- scripts/traffic-recorder/main.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/traffic-recorder/main.rb b/scripts/traffic-recorder/main.rb index 8761f57c..0c33425a 100644 --- a/scripts/traffic-recorder/main.rb +++ b/scripts/traffic-recorder/main.rb @@ -15,7 +15,7 @@ LOG "TrafficRecorder inserted totally #{inserts_total} new traffic records", 'TrafficRecorder' sleep($ione_conf['TrafficRecorder']['check-period']) rescue => e - LOG "TrafficRecorder Error, code: #{e.message}\TrafficRecorder is down now", 'TrafficRecorder' + LOG "TrafficRecorder Error, code: #{e.message}\nTrafficRecorder is down now", 'TrafficRecorder' sleep(30) end end From 2c9645354bdcbb1b8916505a6073e28a52453ff6 Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 4 Jan 2021 15:38:58 +0200 Subject: [PATCH 67/87] new model and zero record hook --- hooks/insert_zero_traffic_record.rb | 43 +++++++++++++++++++++++++++++ rake/set_hooks.rake | 7 +++++ service/records/traffic_records.rb | 12 +++++--- 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 hooks/insert_zero_traffic_record.rb diff --git a/hooks/insert_zero_traffic_record.rb b/hooks/insert_zero_traffic_record.rb new file mode 100644 index 00000000..3556a282 --- /dev/null +++ b/hooks/insert_zero_traffic_record.rb @@ -0,0 +1,43 @@ +#!/usr/bin/env ruby +# -------------------------------------------------------------------------- # +# Copyright 2020, IONe Cloud Project, Support.by # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# -------------------------------------------------------------------------- # + +require 'base64' +require 'nokogiri' + +xml = Nokogiri::XML(Base64::decode64(ARGV.first)) +unless xml.xpath("/CALL_INFO/RESULT").text.to_i == 1 then + puts "VM wasn't allocated, skipping" + exit 0 +end + +vmid = xml.xpath('//EXTRA/VM') + +require 'yaml' +require 'json' +require 'sequel' + +$ione_conf = YAML.load_file("/etc/one/ione.conf") # IONe configuration constants + +require $ione_conf['DB']['adapter'] +$db = Sequel.connect({ + adapter: $ione_conf['DB']['adapter'].to_sym, + user: $ione_conf['DB']['user'], password: $ione_conf['DB']['pass'], + database: $ione_conf['DB']['database'], host: $ione_conf['DB']['host'] }) + +$db[:traffic_records].insert( + vm: vmid, rx: "0", tx: "0", rx_last: "0", tx_last: "0", stime: Time.now.to_i +) \ No newline at end of file diff --git a/rake/set_hooks.rake b/rake/set_hooks.rake index 84bfbb7e..c31c86cb 100644 --- a/rake/set_hooks.rake +++ b/rake/set_hooks.rake @@ -9,6 +9,13 @@ require 'opennebula' "COMMAND" => '/usr/lib/one/ione/hooks/set_price.rb', "ARGUMENTS" => '$API' }, + { + "NAME" => 'insert-zero-traffic-record', + "TYPE" => 'api', + "CALL" => 'one.vm.allocate', + "COMMAND" => '/usr/lib/one/ione/hooks/insert_zero_traffic_record.rb', + "ARGUMENTS" => '$API' + }, { "NAME" => 'pending', "ON" => "CUSTOM", diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index e54e46f8..54447b23 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -1,10 +1,13 @@ begin $db.create_table :traffic_records do primary_key :key - foreign_key :vm, :vm_pool, null: false - Integer :ts, null: false - String :rx, null: false - String :tx, null: false + foreign_key :vm, :vm_pool, null: false + String :rx, null: false + String :rx_last, null: false + String :tx, null: false + String :tx_last, null: false + Integer :stime, null: false + Integer :etime end rescue puts "Table :traffic_records already exists, skipping" @@ -34,6 +37,7 @@ def initialize id, nosync = false end def sync vm = nil + return 0 inserts = 0 if vm.nil? then From 10d3d93e94c4cb2f6a3e8560bcfa95e3656d56c2 Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 4 Jan 2021 16:33:31 +0200 Subject: [PATCH 68/87] new traffic records synchroniser method --- service/records/traffic_records.rb | 60 ++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index 54447b23..edc8b0ba 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -23,6 +23,22 @@ def sorter def mod st st[:traffic] = 0 end + + def conv_keys + [:rx, :tx, :rx_last, :tx_last] + end + def to_i + for key in conv_keys do + @values[key] = @values[key].to_i + end + self + end + def to_s + for key in conv_keys do + @values[key] = @values[key].to_s + end + self + end end class OpenNebula::TrafficRecords < RecordsSource @@ -37,37 +53,57 @@ def initialize id, nosync = false end def sync vm = nil - return 0 - inserts = 0 - if vm.nil? then vm = onblock :vm, @id vm.info! end + last = TrafficRecord.where(vm: vm.id).order(Sequel.asc(:stime)).last + if last.nil? then + return 0 + end + last = last.to_i + + def next? ts, last # Filter expression + return !last[:etime].nil? && ts <= last[:etime] + end + mon_raw = vm.monitoring(['NETTX', 'NETRX']) mon = {} mon_raw['NETTX'].each do | el | + el[0] = el[0].to_i + next if next? el.first, last # Filter records which have been counted mon[el.first] = {} mon[el.first][:tx] = el.last end mon_raw['NETRX'].each do | el | + el[0] = el[0].to_i + next if next? el.first, last # Filter records which have been counted mon[el.first][:rx] = el.last - end + end - last = TrafficRecord.where(vm: vm.id).order(Sequel.asc(:ts)).last - last = { ts: 0 } if last.nil? + return 0 if mon.keys.size == 0 for ts, data in mon do - ts = ts.to_i - if ts > last[:ts] then - args = data.merge(vm: vm.id, ts: ts) - TrafficRecord.insert **args - inserts += 1 + data[:rx], data[:tx] = data[:rx].to_i, data[:tx].to_i + last = last.to_i + + if last[:rx_last] > data[:rx] || last[:tx_last] > data[:tx] then + last[:rx] += data[:rx] + last[:tx] += data[:tx] + else + last[:rx] += (data[:rx] - last[:rx_last]) + last[:tx] += (data[:tx] - last[:tx_last]) end + + last[:rx_last], last[:tx_last] = data[:rx], data[:tx] + last[:etime] = ts end - inserts + last = last.to_s + TrafficRecord.where(stime: last[:stime]).update(**last.values) + + mon.keys.size end def find stime, etime From ef7577fea78cd1d6018b5ebfe79044ddce830918 Mon Sep 17 00:00:00 2001 From: Mik Date: Mon, 4 Jan 2021 17:52:53 +0200 Subject: [PATCH 69/87] sorter and modificator --- service/records/traffic_records.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index edc8b0ba..f48dc55e 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -18,10 +18,10 @@ def sortable self end def sorter - ts + etime end def mod st - st[:traffic] = 0 + st.merge! rx: rx, tx: tx end def conv_keys From 210019e5af522ccab1b7fa4be9b1551267c1b331 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 11:34:11 +0200 Subject: [PATCH 70/87] TrafficRecords source find method --- service/records/traffic_records.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index f48dc55e..ed14a810 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -106,8 +106,16 @@ def next? ts, last # Filter expression mon.keys.size end - def find stime, etime - @records.where{ ts < stime }.order(Sequel.asc :ts).limit(1) + def find st, et + last = TrafficRecord.where(vm: @id).order(Sequel.asc(:stime)).last + if last[:etime] - last[:stime] >= 86400 then # If record is elder than 24 hours + args = last.values.without(:key, :rx, :tx, :stime) + args.merge! rx: 0, tx: 0, stime: args[:etime] # Setting up new record with zero rx, tx and same rx_last, tx_last + TrafficRecord.insert(**args.without(:etime)) + end + + @records.exclude(etime: nil).exclude{ etime - stime < 86400 }.where(etime: st..et) + end end end \ No newline at end of file From 516c8f58d875790c78f215bc6c871f2bc331ebbc Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 11:39:57 +0200 Subject: [PATCH 71/87] traffic init_state --- service/records/traffic_records.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index ed14a810..f6c5264d 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -116,6 +116,14 @@ def find st, et @records.exclude(etime: nil).exclude{ etime - stime < 86400 }.where(etime: st..et) end + + def init_state stime + state = { rx: 0, tx: 0 } + rec = TrafficRecord.where(vm: @id).where(stime: stime).all.last + unless rec.nil? then + state[:rx], state[:tx] = rec.rx, rec.tx + end + state end end \ No newline at end of file From 6ba612f47a5e5798ca218dd3fad54f7cae1a0086 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 11:41:37 +0200 Subject: [PATCH 72/87] ints are better, right? --- service/records/traffic_records.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index f6c5264d..f39f9f84 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -121,6 +121,7 @@ def init_state stime state = { rx: 0, tx: 0 } rec = TrafficRecord.where(vm: @id).where(stime: stime).all.last unless rec.nil? then + rec = rec.to_i state[:rx], state[:tx] = rec.rx, rec.tx end state From 0ad3d960b46073becc151a1ea2e448ab02faac0f Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 11:44:43 +0200 Subject: [PATCH 73/87] TRAFFIC_COST to settings table --- models/SettingsDriver.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/SettingsDriver.rb b/models/SettingsDriver.rb index c6ad2a57..b72e08a0 100644 --- a/models/SettingsDriver.rb +++ b/models/SettingsDriver.rb @@ -17,7 +17,8 @@ ['PUBLIC_IP_COST', "0.0", "Public IP Address cost", 0, "num"], ['PUBLIC_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Public Network Pool ID", 1, "object"], ['PRIVATE_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Private Network Pool ID", 1, "object"], - ['CURRENCY_MAIN', "€", "Currency", 0, "str"] + ['CURRENCY_MAIN', "€", "Currency", 0, "str"], + ['TRAFFIC_COST', "0.0", "Cost per 1 kByte traffic", 1, "num"] ] required.each do | record | $db[:settings].insert(name: record[0], body: record[1], description: record[2], access_level: record[3], type: record[4]) From 76c36b3194edf14918eacf499093b7936832eeac Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:01:21 +0200 Subject: [PATCH 74/87] i didn't mean it --- service/records/traffic_records.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index f39f9f84..88f63526 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -119,12 +119,11 @@ def find st, et def init_state stime state = { rx: 0, tx: 0 } - rec = TrafficRecord.where(vm: @id).where(stime: stime).all.last + rec = TrafficRecord.where(vm: @id).where(etime: stime).all.last unless rec.nil? then rec = rec.to_i state[:rx], state[:tx] = rec.rx, rec.tx end state end - end \ No newline at end of file From 09f8dcb79291c8df8449dfbdb9e86b5cfd98ff89 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:07:05 +0200 Subject: [PATCH 75/87] hyper important alias --- service/records/traffic_records.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index 88f63526..828cd5ae 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -20,6 +20,8 @@ def sortable def sorter etime end + alias :ts :sorter + def mod st st.merge! rx: rx, tx: tx end From 0237165efd80d6140e8e3a55a82f84534c7a7b18 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:07:17 +0200 Subject: [PATCH 76/87] ints are always better --- service/records/traffic_records.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/records/traffic_records.rb b/service/records/traffic_records.rb index 828cd5ae..ccccad3d 100644 --- a/service/records/traffic_records.rb +++ b/service/records/traffic_records.rb @@ -23,7 +23,7 @@ def sorter alias :ts :sorter def mod st - st.merge! rx: rx, tx: tx + st.merge! rx: rx.to_i, tx: tx.to_i end def conv_keys From 64570119961680e58d13e26a5199b12cacd65f9a Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:07:58 +0200 Subject: [PATCH 77/87] quite new params form for bill method --- service/biller.rb | 2 +- service/billers/capacity.rb | 2 +- service/billers/disk.rb | 2 +- service/billers/snapshots.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/service/biller.rb b/service/biller.rb index 9f294666..65d1480c 100644 --- a/service/biller.rb +++ b/service/biller.rb @@ -18,7 +18,7 @@ def billing_period @vm['//BILLING_PERIOD'] end - def bill bill, state, delta + def bill bill:, state:, delta:, record: nil 0 end end diff --git a/service/billers/capacity.rb b/service/billers/capacity.rb index 78801d6c..c553730a 100644 --- a/service/billers/capacity.rb +++ b/service/billers/capacity.rb @@ -15,7 +15,7 @@ def check_biller return false end - def bill bill, state, delta + def bill bill:, state:, delta:, record: nil if state[:state] == 'on' || billing_period != 'PAYG' then bill[:capacity] = delta * @cost end diff --git a/service/billers/disk.rb b/service/billers/disk.rb index 008839d4..6cb6db0f 100644 --- a/service/billers/disk.rb +++ b/service/billers/disk.rb @@ -18,7 +18,7 @@ def check_biller return false end - def bill bill, state, delta + def bill bill:, state:, delta:, record: nil bill[:disk] = delta * @cost bill end diff --git a/service/billers/snapshots.rb b/service/billers/snapshots.rb index f8924bb1..24493cbb 100644 --- a/service/billers/snapshots.rb +++ b/service/billers/snapshots.rb @@ -9,7 +9,7 @@ def check_biller return false end - def bill bill, state, delta + def bill bill:, state:, delta:, record: nil bill[:snapshots] = delta * @cost * state[:snaps] bill end From 4a1d47f2fb3ed2624d4234e979263ea9610b5f41 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:08:41 +0200 Subject: [PATCH 78/87] check_biller with teaser --- service/billers/traffic.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/service/billers/traffic.rb b/service/billers/traffic.rb index 6ef5b400..4b142859 100644 --- a/service/billers/traffic.rb +++ b/service/billers/traffic.rb @@ -4,7 +4,13 @@ def check_biller return false if @cost.nil? @cost = @cost.to_f - return @cost > 0 + @costs = { rx: @cost, tx: @cost } # Will Add support for differnt rx and tx prices + + return @costs.values.inject(0) do | r, c | + r += c.to_f + rescue + r + end > 0 rescue return false end From 9cf830bfca6c7a5c65fb3aac45e6c881436584e1 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:08:53 +0200 Subject: [PATCH 79/87] TrafficBiller bill method --- service/billers/traffic.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/service/billers/traffic.rb b/service/billers/traffic.rb index 4b142859..6bd31e3d 100644 --- a/service/billers/traffic.rb +++ b/service/billers/traffic.rb @@ -14,4 +14,12 @@ def check_biller rescue return false end + + def bill bill:, state:, delta:, record: + if record.class == TrafficRecord then + bill[:rx] = state[:rx] / 1000 * @costs[:rx] + bill[:tx] = state[:tx] / 1000 * @costs[:tx] + end + bill + end end \ No newline at end of file From 9ee62d64b2848fcf41cba4f0fd10231f142f0b3a Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:09:23 +0200 Subject: [PATCH 80/87] Using new bill method params --- service/showback.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/showback.rb b/service/showback.rb index dee19d1a..5f54184e 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -92,7 +92,7 @@ def make_bill curr.mod state bill_rec = {time: con.ts} @billers.each do | biller | - bill_rec.merge! biller.bill(bill_rec, state, delta) + bill_rec.merge! biller.bill(bill: bill_rec, state: state, delta: delta, record: curr) end @bill << bill_rec end From 545b6ee457d19074a6109baa9419a83ab67e3d06 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:09:34 +0200 Subject: [PATCH 81/87] Using TrafficBiller in Billing class --- service/showback.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/showback.rb b/service/showback.rb index 5f54184e..0d44c0db 100644 --- a/service/showback.rb +++ b/service/showback.rb @@ -7,7 +7,7 @@ class Timeline attr_reader :vm, :stime, :etime, :group_by_day, :timeline, :sources, :compiled, :state SOURCES = [ - Records, SnapshotRecords + Records, SnapshotRecords, TrafficRecords ] def initialize vm, stime, etime, group_by_day = false @@ -73,7 +73,7 @@ class Billing attr_reader :timeline, :bill BILLERS = [ - CapacityBiller, DiskBiller, SnapshotsBiller + CapacityBiller, DiskBiller, SnapshotsBiller, TrafficBiller ] def initialize vm, stime, etime From 19a7833f258d2c1b62bf18b37d45190217a814ce Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:24:40 +0200 Subject: [PATCH 82/87] we have floats here --- service/billers/traffic.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/billers/traffic.rb b/service/billers/traffic.rb index 6bd31e3d..fff56062 100644 --- a/service/billers/traffic.rb +++ b/service/billers/traffic.rb @@ -17,8 +17,8 @@ def check_biller def bill bill:, state:, delta:, record: if record.class == TrafficRecord then - bill[:rx] = state[:rx] / 1000 * @costs[:rx] - bill[:tx] = state[:tx] / 1000 * @costs[:tx] + bill[:rx] = state[:rx] / 1000.0 * @costs[:rx] + bill[:tx] = state[:tx] / 1000.0 * @costs[:tx] end bill end From 6cb553705348d99e8380aaff41804da480798258 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:25:13 +0200 Subject: [PATCH 83/87] VM drives method --- service/objects/vm.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/service/objects/vm.rb b/service/objects/vm.rb index 0824e525..bf0da237 100644 --- a/service/objects/vm.rb +++ b/service/objects/vm.rb @@ -428,4 +428,11 @@ def initialize params = [] super "#{params[0]}\nParams:#{@params.inspect}" end end + + # List VM Drives + # @return [Array] + def drives + r = to_hash!['VM']['TEMPLATE']['DISK'] + r.class == Array ? r : [ r ] + end end \ No newline at end of file From ad084a136f7421db2e8e536ecfc40d5c50e72cd6 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:29:04 +0200 Subject: [PATCH 84/87] correct drives billing --- service/billers/disk.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/billers/disk.rb b/service/billers/disk.rb index 6cb6db0f..8f235f06 100644 --- a/service/billers/disk.rb +++ b/service/billers/disk.rb @@ -13,13 +13,14 @@ def check_biller return false if r <= 0 @cost = @costs[@vm['/VM/USER_TEMPLATE/DRIVE']].to_f + @size = vm.drives.inject(0) { | r, d | r += d['SIZE'].to_i } / 1000.0 return @cost > 0 rescue return false end def bill bill:, state:, delta:, record: nil - bill[:disk] = delta * @cost + bill[:disk] = delta * @cost * @size bill end end \ No newline at end of file From 4913d7485a10da0ef82f06e5cdeaa889a40fcf1e Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:29:29 +0200 Subject: [PATCH 85/87] Traffic cost with description in default settings --- models/SettingsDriver.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/SettingsDriver.rb b/models/SettingsDriver.rb index b72e08a0..a19e26d8 100644 --- a/models/SettingsDriver.rb +++ b/models/SettingsDriver.rb @@ -18,7 +18,7 @@ ['PUBLIC_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Public Network Pool ID", 1, "object"], ['PRIVATE_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Private Network Pool ID", 1, "object"], ['CURRENCY_MAIN', "€", "Currency", 0, "str"], - ['TRAFFIC_COST', "0.0", "Cost per 1 kByte traffic", 1, "num"] + ['TRAFFIC_COST', "0.0", "Cost per 1 kByte traffic", 1, "num"], ] required.each do | record | $db[:settings].insert(name: record[0], body: record[1], description: record[2], access_level: record[3], type: record[4]) From 7abc18a026e6ffb6ae7bfac8614f8e4f433ac330 Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:29:41 +0200 Subject: [PATCH 86/87] Snapshot cost in default settings --- models/SettingsDriver.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/models/SettingsDriver.rb b/models/SettingsDriver.rb index a19e26d8..b5b45a8e 100644 --- a/models/SettingsDriver.rb +++ b/models/SettingsDriver.rb @@ -19,6 +19,7 @@ ['PRIVATE_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Private Network Pool ID", 1, "object"], ['CURRENCY_MAIN', "€", "Currency", 0, "str"], ['TRAFFIC_COST', "0.0", "Cost per 1 kByte traffic", 1, "num"], + ['SNAPSHOT_COST', "0.0", "Cost 1 Snapshot per sec", 1, "num"] ] required.each do | record | $db[:settings].insert(name: record[0], body: record[1], description: record[2], access_level: record[3], type: record[4]) From 9057d4283c2618e29ddd516ef8a92c33e38322ee Mon Sep 17 00:00:00 2001 From: Mik Date: Tue, 5 Jan 2021 12:30:10 +0200 Subject: [PATCH 87/87] costs descriptions --- models/SettingsDriver.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/SettingsDriver.rb b/models/SettingsDriver.rb index b5b45a8e..35caf48a 100644 --- a/models/SettingsDriver.rb +++ b/models/SettingsDriver.rb @@ -9,12 +9,12 @@ required = [ ['ALERT', "0.0", "Balance, when user will be alerted", 0, "num"], - ['CAPACITY_COST', "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}", "VM Capacity resources costs", 1, "object"], + ['CAPACITY_COST', "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}", "VM Capacity resources costs per sec", 1, "object"], ['DISK_TYPES', "HDD,SSD,NVMe", "Comma-separated list of existing disk types", 1, "list"], - ['DISK_COSTS', "{\"disk_type\":\"price\"}", "Costs of different disk types", 1, "object"], + ['DISK_COSTS', "{\"disk_type\":\"price\"}", "Costs of different disk types GB/sec", 1, "object"], ['IAAS_GROUP_ID', 'iaas_group_id', "IaaS(VDC) Users group ID", 1, "num"], ['NODES_DEFAULT', "{\"hypervisor_name\":\"host_id\"}", "Default nodes for different hypervisors", 1, "object"], - ['PUBLIC_IP_COST', "0.0", "Public IP Address cost", 0, "num"], + ['PUBLIC_IP_COST', "0.0", "Public IP Address cost per sec", 0, "num"], ['PUBLIC_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Public Network Pool ID", 1, "object"], ['PRIVATE_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Private Network Pool ID", 1, "object"], ['CURRENCY_MAIN', "€", "Currency", 0, "str"],