New easy-customisable Showback engine

docs/Showback.md

@@ -0,0 +1,8 @@
+# Notes about configuring VM Showback
+
+## BILLING_PERIOD
+
+Each and every VM should have BILLING_PERIOD defined in template.
+Possible values are:
+ - PAYG - VM will be billed per second
+ - PRE_<% n_days %> - Pre-Paid per N days, e.g. `PRE_30`, then full Capacity and Disk cost will be charged on deploy, and then every 30 days.

hooks/insert_zero_traffic_record.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+# -------------------------------------------------------------------------- #
+# Copyright 2020, IONe Cloud Project, Support.by                             #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+# -------------------------------------------------------------------------- #
+
+require 'base64'
+require 'nokogiri'
+
+xml = Nokogiri::XML(Base64::decode64(ARGV.first))
+unless xml.xpath("/CALL_INFO/RESULT").text.to_i == 1 then
+    puts "VM wasn't allocated, skipping"
+    exit 0
+end
+
+vmid = xml.xpath('//EXTRA/VM')
+
+require 'yaml'
+require 'json'
+require 'sequel'
+
+$ione_conf = YAML.load_file("/etc/one/ione.conf") # IONe configuration constants
+
+require $ione_conf['DB']['adapter']
+$db = Sequel.connect({
+        adapter: $ione_conf['DB']['adapter'].to_sym,
+        user: $ione_conf['DB']['user'], password: $ione_conf['DB']['pass'],
+        database: $ione_conf['DB']['database'], host: $ione_conf['DB']['host']  })
+
+$db[:traffic_records].insert(
+    vm: vmid, rx: "0", tx: "0", rx_last: "0", tx_last: "0", stime: Time.now.to_i
+)

ione_server.rb

@@ -115,6 +115,8 @@ class Settings < Sequel::Model(:settings); end
 puts 'Including on_helper funcs'
 require "#{ROOT}/service/on_helper.rb"
 include ONeHelper
+puts 'Including showback'
+require "#{ROOT}/service/showback.rb"
 puts 'Including Deferable module'
 require "#{ROOT}/service/defer.rb"
 

models/SettingsDriver.rb

@@ -9,15 +9,17 @@
 
     required = [
         ['ALERT', "0.0", "Balance, when user will be alerted", 0, "num"],
-        ['CAPACITY_COST', "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}", "VM Capacity resources costs", 1, "object"],
+        ['CAPACITY_COST', "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}", "VM Capacity resources costs per sec", 1, "object"],
         ['DISK_TYPES', "HDD,SSD,NVMe", "Comma-separated list of existing disk types", 1, "list"],
-        ['DISK_COSTS', "{\"disk_type\":\"price\"}", "Costs of different disk types", 1, "object"],
+        ['DISK_COSTS', "{\"disk_type\":\"price\"}", "Costs of different disk types GB/sec", 1, "object"],
         ['IAAS_GROUP_ID', 'iaas_group_id', "IaaS(VDC) Users group ID", 1, "num"],
         ['NODES_DEFAULT', "{\"hypervisor_name\":\"host_id\"}", "Default nodes for different hypervisors", 1, "object"],
-        ['PUBLIC_IP_COST', "0.0", "Public IP Address cost", 0, "num"],
+        ['PUBLIC_IP_COST', "0.0", "Public IP Address cost per sec", 0, "num"],
         ['PUBLIC_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Public Network Pool ID", 1, "object"],
         ['PRIVATE_NETWORK_DEFAULTS', "{\"NETWORK_ID\":\"network_id\"}", "Default Private Network Pool ID", 1, "object"],
-        ['CURRENCY_MAIN', "€", "Currency", 0, "str"]
+        ['CURRENCY_MAIN', "€", "Currency", 0, "str"],
+        ['TRAFFIC_COST', "0.0", "Cost per 1 kByte traffic", 1, "num"],
+        ['SNAPSHOT_COST', "0.0", "Cost 1 Snapshot per sec", 1, "num"]
     ]
     required.each do | record |
         $db[:settings].insert(name: record[0], body: record[1], description: record[2], access_level: record[3], type: record[4])

rake/set_hooks.rake

@@ -9,6 +9,13 @@ require 'opennebula'
         "COMMAND" => '/usr/lib/one/ione/hooks/set_price.rb',
         "ARGUMENTS" => '$API'
     },
+    {
+        "NAME" => 'insert-zero-traffic-record',
+        "TYPE" => 'api',
+        "CALL" => 'one.vm.allocate',
+        "COMMAND" => '/usr/lib/one/ione/hooks/insert_zero_traffic_record.rb',
+        "ARGUMENTS" => '$API'
+    },
     {
         "NAME" => 'pending',
         "ON" => "CUSTOM",

rake/tests/Rakefile

@@ -0,0 +1,17 @@
+task :before_test do
+    require '/usr/lib/one/ione/ione_server.rb'
+end
+
+def passed
+    puts "--- " + "Passed".green
+end
+def fail msg
+    puts msg.red
+    exit
+end
+def warn msg
+    puts msg.yellow
+end
+
+load "rake/tests/showback.rake"
+

rake/tests/showback.rake

@@ -0,0 +1,68 @@
+desc "Test Capacity Showback"
+task :cap_showback_test => :before_test do
+    puts "\n####################\n# Testing capacity #\n####################"
+
+    $db[:settings].where(name: "CAPACITY_COST").update(body: "{\"CPU_COST\":\"0.5\",\"MEMORY_COST\":\"0.5\"}")
+    $db[:settings].where(name: "DISK_COSTS").update(body: "{\"HDD\":\"0.0\"}")
+
+    r = onblock(:vm, 3).calculate_showback 0, 1200
+    if r[:TOTAL] == 600 then
+        passed
+    else
+        warn "#{0}, #{1200} => #{r[:TOTAL]}"
+    end
+    r = onblock(:vm, 3).calculate_showback 300, 1200
+    if r[:TOTAL] == 300 then
+        passed
+    else
+        warn "#{300}, #{1200} => #{r[:TOTAL]}"
+    end
+    r = onblock(:vm, 3).calculate_showback 300, 500
+    if r[:TOTAL] == 200 then
+        passed
+    else
+        warn "#{300}, #{500} => #{r[:TOTAL]}"
+    end
+    r = onblock(:vm, 3).calculate_showback 300, 1400
+    if r[:TOTAL] == 500 then
+        passed
+    else
+        warn "#{300}, #{1400} => #{r[:TOTAL]}"
+    end
+end
+
+desc "Test Disk Showback"
+task :disk_showback_test => :before_test do
+    puts "\n################\n# Testing Disk #\n################"
+
+    $db[:settings].where(name: "CAPACITY_COST").update(body: "{\"CPU_COST\":\"0.0\",\"MEMORY_COST\":\"0.0\"}")
+    $db[:settings].where(name: "DISK_COSTS").update(body: "{\"HDD\":\"1\"}")
+
+    r = onblock(:vm, 3).calculate_showback 0, 1200
+    if r[:TOTAL] == 1200 then
+        passed
+    else
+        warn "#{0}, #{1200} => #{r[:TOTAL]}"
+    end
+    r = onblock(:vm, 3).calculate_showback 300, 1200
+    if r[:TOTAL] == 900 then
+        passed
+    else
+        warn "#{300}, #{1200} => #{r[:TOTAL]}"
+    end
+    r = onblock(:vm, 3).calculate_showback 300, 500
+    if r[:TOTAL] == 200 then
+        passed
+    else
+        warn "#{300}, #{500} => #{r[:TOTAL]}"
+    end
+    r = onblock(:vm, 3).calculate_showback 300, 1400
+    if r[:TOTAL] == 1100 then
+        passed
+    else
+        warn "#{300}, #{1400} => #{r[:TOTAL]}"
+    end
+end
+
+desc "Showback Tests"
+task :showback_test => [:cap_showback_test, :disk_showback_test] do; end

scripts/traffic-recorder/main.rb

@@ -0,0 +1,24 @@
+begin
+    LOG 'Traffic Recorder has been initialized', 'TrafficRecorder'
+    loop do
+        begin
+            vm_pool = VirtualMachinePool.new($client)
+            vm_pool.info_all
+            inserts_total = 0
+
+            vm_pool.each do | vm |
+                inserts = TrafficRecords.new(vm.id, true).sync vm        
+                LOG "VM #{vm.id}: Inserted #{inserts} new traffic records", "TrafficRecorder"
+                inserts_total += inserts
+            end
+
+            LOG "TrafficRecorder inserted totally #{inserts_total} new traffic records", 'TrafficRecorder'
+            sleep($ione_conf['TrafficRecorder']['check-period'])
+        rescue => e
+            LOG "TrafficRecorder Error, code: #{e.message}\nTrafficRecorder is down now", 'TrafficRecorder'
+            sleep(30)
+        end
+    end
+rescue
+    LOG "TrafficRecorder fatal error, service is crashed", 'TrafficRecorderThread'
+end

service/biller.rb

@@ -0,0 +1,26 @@
+class Biller
+
+    def initialize vm
+        @vm = vm
+    end
+
+    # Costs hash
+    def costs
+        SETTINGS_TABLE.as_hash(:name, :body).select {|key| key.include? 'COST' }
+    end
+
+    # Check if this biller should be used in Billing
+    def check_biller
+        true
+    end
+
+    def billing_period
+        @vm['//BILLING_PERIOD']
+    end
+
+    def bill bill:, state:, delta:, record: nil
+        0
+    end
+end
+
+Dir["#{ROOT}/service/billers/*.rb"].each {|file| require file }

service/billers/capacity.rb

@@ -0,0 +1,24 @@
+class CapacityBiller < Biller
+    # Checking if Capacity costs are given, otherwise there is no point to calculate it
+    def check_biller
+        @costs = JSON.parse(costs['CAPACITY_COST'])
+        return false if @costs.nil?
+
+        @cost = 
+            @costs.values.inject(0) do | r, c |
+                r += c.to_f
+            rescue
+                r
+            end
+        return @cost > 0
+    rescue
+        return false
+    end
+
+    def bill bill:, state:, delta:, record: nil
+        if state[:state] == 'on' || billing_period != 'PAYG' then
+            bill[:capacity] = delta * @cost
+        end
+        bill
+    end
+end

service/billers/disk.rb

@@ -0,0 +1,26 @@
+class DiskBiller < Biller
+    # Checking if Capacity costs are given, otherwise there is no point to calculate it
+    def check_biller
+        @costs = JSON.parse(costs['DISK_COSTS'])
+        return false if @costs.nil?
+
+        r = 
+            @costs.values.inject(0) do | r, c |
+                r += c.to_f
+            rescue
+                r
+            end
+        return false if r <= 0
+
+        @cost = @costs[@vm['/VM/USER_TEMPLATE/DRIVE']].to_f
+        @size = vm.drives.inject(0) { | r, d | r += d['SIZE'].to_i } / 1000.0
+        return @cost > 0
+    rescue
+        return false
+    end
+
+    def bill bill:, state:, delta:, record: nil
+        bill[:disk] = delta * @cost * @size
+        bill
+    end
+end

service/billers/snapshots.rb

@@ -0,0 +1,16 @@
+class SnapshotsBiller < Biller
+    def check_biller
+        @cost = JSON.parse(costs['SNAPSHOT_COST'])
+        return false if @cost.nil?
+
+        @cost = @cost.to_f
+        return @cost > 0
+    rescue
+        return false
+    end
+
+    def bill bill:, state:, delta:, record: nil
+        bill[:snapshots] = delta * @cost * state[:snaps]
+        bill
+    end
+end

service/billers/traffic.rb

@@ -0,0 +1,25 @@
+class TrafficBiller < Biller
+    def check_biller
+        @cost = JSON.parse(costs['TRAFFIC_COST'])
+        return false if @cost.nil?
+
+        @cost = @cost.to_f
+        @costs = { rx: @cost, tx: @cost } # Will Add support for differnt rx and tx prices
+
+        return @costs.values.inject(0) do | r, c |
+            r += c.to_f
+        rescue
+            r
+        end > 0
+    rescue
+        return false
+    end
+
+    def bill bill:, state:, delta:, record: 
+        if record.class == TrafficRecord then
+            bill[:rx] = state[:rx] / 1000.0 * @costs[:rx]
+            bill[:tx] = state[:tx] / 1000.0 * @costs[:tx]
+        end
+        bill
+    end
+end