Umbu and Threads versions are now available and new flags

.gitignore

@@ -1,5 +1,7 @@
-run
 .vscode/
 out/
 *.o
-test.sh
+*.tif
+*.out
+test.sh
+run 

Makefile

@@ -1,21 +1,22 @@
 CC=g++
 CFLAGS = -std=c++14
+LIBS = -lpthread -ltiff
 
 DEPS = promethee/*.h
 
-SRCS = $(shell find -name *.cpp)
+SRCS = $(shell find promethee -name *.cpp)
 OBJS := $(addsuffix .o,$(basename $(SRCS)))
 
 all: run
 
 promethee/%.o: promethee/%.cpp $(DEPS)
-	$(CC) -c -o $@ $< $(CFLAGS)
+	$(CC) -c -o $@ $< $(CFLAGS) $(LIBS)
 
 promethee/vanilla/%.o: promethee/vanilla/%.cpp $(DEPS)
-	$(CC) -c -o $@ $< $(CFLAGS)
+	$(CC) -c -o $@ $< $(CFLAGS) $(LIBS)
 
 run: $(OBJS)
-	$(CC) -o $@ $^ 
+	$(CC) -o $@ $^ $(LIBS)
 
 clean: 
 	rm -rf $(OBJS)

caja/playground/monitor/collect-cpu-usage.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export LC_ALL="C"
+TIME_BETWEEN_COMMANDS=1
+echo TIMESTAMP, USER, NICE, SYS, IOWAIT, IRQ, SOFT, STEAL, GUEST, GNICE, IDLE
+while [ -e /proc/$1 ]; do
+ echo "foo" >> /tmp/baa  
+/usr/bin/mpstat $TIME_BETWEEN_COMMANDS 1 | grep -e '[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}[ \t]*'all | awk -v date="$( date +"%s" )" '{ print date", "$3", "$4", "$5", "$6", "$7", "$8", "$9", "$10", "$11", "$12 }' 2> /dev/null
+done

caja/playground/monitor/collect-disk-usage.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+export LC_ALL="C"
+TIME_BETWEEN_COMMANDS=1
+echo TIMESTAMP, PID, COMMAND, KB_READ/S, KB_WRITE/S
+while [ -e /proc/$1 ]; do
+  sudo /usr/bin/pidstat -d $TIME_BETWEEN_COMMANDS 1 | grep -e '[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}[ \t]*[0-9]' | awk -v date="$( date +"%s" )" '{ if ($8) $7=$8; if (NF) print date", "$3", "$7", "$4", "$5 ; }' 2> /dev/null
+done

caja/playground/monitor/collect-memory-usage.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export LC_NUMERIC="C"
+TIME_BETWEEN_COMMANDS=1
+echo TIMESTAMP, TOTAL, USED, FREE, SHARED, BUFFER/CACHE, AVAILABLE
+while [ -e /proc/$1 ]; do
+  free -k | grep Mem: | awk -v date="$( date +"%s" )" '{ print date", "$2", "$3", "$4", "$5", "$6", "$7 }' 2> /dev/null
+  sleep $TIME_BETWEEN_COMMANDS
+done

caja/playground/monitor/filter-sum-disk-usage.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Script to filter and sum the instances of process made by RScript in the same second
+
+import sys
+
+def main():
+	args = sys.argv
+
+	disk_usage_file = open(args[1])
+	filtered_disk_usage = open(args[2], "w+")
+
+	wanted_head = ["TIMESTAMP", "KB_READ/S", "KB_WRITE/S\n"]
+	a_head = disk_usage_file.readline().rsplit(", ")
+	head = ""
+	for h in a_head:
+		if(h in wanted_head):
+			head += h + ", "
+
+	head = head[0 : len(head) - 2]
+
+	filtered_disk_usage.write(head)
+
+	filtered_disk_usage_dict = {}
+
+	for line in disk_usage_file:
+		a_line = line.rsplit(", ")
+
+		ts = a_line[0]
+
+		cmd = a_line[2]
+
+		kb_rw = [float(a_line[3]), float(a_line[4])]
+
+		if(cmd == "R"):
+			if(ts in filtered_disk_usage_dict.keys()):
+				filtered_disk_usage_dict[ts][0] += kb_rw[0]
+				filtered_disk_usage_dict[ts][1] += kb_rw[1]
+			else:
+				filtered_disk_usage_dict[ts] = kb_rw
+
+	filtered_disk_usage_list = list(filtered_disk_usage_dict.keys())
+	filtered_disk_usage_list.sort()
+
+	for x in filtered_disk_usage_list:
+		filtered_disk_usage.write(x + ", " + str(filtered_disk_usage_dict[x][0]) + ", " + str(filtered_disk_usage_dict[x][1]) + "\n")
+
+	disk_usage_file.close()
+	filtered_disk_usage.close()
+
+main()

caja/playground/monitor/plot-usages.r

@@ -0,0 +1,125 @@
+#http://ggplot2.tidyverse.org/reference/#section-scales
+
+library("ggplot2")
+library("scales")
+library("data.table")
+library("stringi")
+
+args = commandArgs(trailingOnly=TRUE)
+
+cpu_usage_path = args[1]
+mem_usage_path = args[2]
+disk_usage_path = args[3]
+proctimes_path = args[4]
+
+cpu_usage = read.table(sep = ",", cpu_usage_path, header=TRUE)
+mem_usage = read.table(sep = ",", mem_usage_path, header=TRUE)
+disk_usage = read.table(sep = ",", disk_usage_path, header=TRUE)
+proctimes = read.table(sep = " ", proctimes_path, header=FALSE)
+
+# Cada proctime com label deve indicar o fim de o processamento desse label no script
+proctimes = subset(proctimes, stri_length(V2) > 0)
+
+cpu_min_TS = min(cpu_usage$TIMESTAMP)
+mem_min_TS = min(mem_usage$TIMESTAMP)
+disk_min_TS = min(disk_usage$TIMESTAMP)
+proctimes_min_TS = min(proctimes$V1)
+
+start_program_TS = min( c(cpu_min_TS, mem_min_TS, disk_min_TS, proctimes_min_TS) )
+
+cpu_usage$IDLE = 100 - cpu_usage$IDLE
+
+mem_usage$TOTAL = (mem_usage$USED/mem_usage$TOTAL)*100
+
+cpu_usage$TIMESTAMP = cpu_usage$TIMESTAMP - start_program_TS
+mem_usage$TIMESTAMP = mem_usage$TIMESTAMP - start_program_TS
+disk_usage$TIMESTAMP = disk_usage$TIMESTAMP - start_program_TS
+proctimes$V1 = proctimes$V1 - start_program_TS
+
+cpu_usage = cpu_usage[c("TIMESTAMP", "IDLE", "GNICE")]
+mem_usage = mem_usage[c("TIMESTAMP", "TOTAL", "BUFFER.CACHE")]
+disk_read_usage = disk_usage[c("TIMESTAMP", "KB_READ.S", "KB_WRITE.S")]#TIMESTAMP, KB/S, TYPE(READ/WRITE)
+disk_write_usage = disk_usage[c("TIMESTAMP", "KB_READ.S", "KB_WRITE.S")]
+
+disk_write_usage$"KB_READ.S" = disk_write_usage$"KB_WRITE.S"
+
+setnames(cpu_usage, c("TIMESTAMP", "USAGE", "TYPE"))
+setnames(mem_usage, c("TIMESTAMP", "USAGE", "TYPE"))
+setnames(disk_read_usage, c("TIMESTAMP", "MB.S", "TYPE"))
+setnames(disk_write_usage, c("TIMESTAMP", "MB.S", "TYPE"))
+
+cpu_usage$TYPE = 'CPU'
+mem_usage$TYPE = 'MEM'
+disk_read_usage$TYPE = 'READ'
+disk_write_usage$TYPE = 'WRITE'
+
+data_cpu_mem = rbind(cpu_usage, mem_usage)
+data_disk = rbind(disk_write_usage, disk_read_usage)
+
+data_disk$TYPE = as.character(data_disk$TYPE)
+data_disk$TYPE = factor(data_disk$TYPE, levels = unique(data_disk$TYPE))
+
+data_disk$MB.S = data_disk$MB.S/1024
+
+med_proctimes = 0
+previous = 0
+
+for(i in 1:dim(proctimes)[1])
+{
+	med_proctimes[i] = (previous + proctimes$V1[i])/2.0
+	previous = proctimes$V1[i]
+}
+
+
+pplot = ggplot(cpu_usage, aes(x=TIMESTAMP, y=USAGE, group=TYPE, colour=TYPE)) +
+	geom_line(size=0.7) + xlab("TIME (s)") + 
+	ylab("USAGE (%)") + 
+	geom_vline(xintercept = proctimes$V1, linetype=2, size=0.4) +
+	annotate("text", x = med_proctimes, y=0.0, label = proctimes$V2) + 
+	scale_colour_manual(values=c("#FF7777")) + 
+	theme_bw() + 
+	guides(fill=guide_legend(title=NULL)) + 
+	theme(legend.title=element_blank())
+
+ggsave("usage_cpu.png", pplot, width=14)
+
+
+pplot = ggplot(mem_usage, aes(x=TIMESTAMP, y=USAGE, group=TYPE, colour=TYPE)) + 
+	geom_line(size=0.7) + xlab("TIME (s)") + 
+	ylab("USAGE (%)") + 
+	geom_vline(xintercept = proctimes$V1, linetype=2, size=0.4) + 
+	annotate("text", x = med_proctimes, y=0.0, label = proctimes$V2) + 
+	scale_colour_manual(values=c("#0066CC")) + 
+	theme_bw() + 
+	guides(fill=guide_legend(title=NULL)) + 
+	theme(legend.title=element_blank())
+
+ggsave("usage_mem.png", pplot, width=14)
+
+
+pplot = ggplot(data_cpu_mem, aes(x=TIMESTAMP, y=USAGE, color=TYPE)) + 
+	geom_line(size=0.7) + 
+	xlab("TIME (s)") + 
+	ylab("USAGE (%)") + 
+	geom_vline(xintercept = proctimes$V1, linetype=2, size=0.4) + 
+	annotate("text", x = med_proctimes, y=0.0, label = proctimes$V2) + 
+	scale_colour_manual(values=c("#FF7777", "#0066CC")) + 
+	theme_bw() + 
+	guides(fill=guide_legend(title=NULL)) + 
+	theme(legend.title=element_blank())
+
+ggsave("usage_cpu_mem.png", pplot, width=14)
+
+
+pplot = ggplot(data_disk, aes(x=TIMESTAMP, y=MB.S, color=TYPE)) + 
+	geom_line(size=0.7) + 
+	xlab("TIME (s)") + 
+	ylab("MB/S") + 
+	geom_vline(xintercept = proctimes$V1, linetype=2, size=0.4) + 
+	annotate("text", x = med_proctimes, y=-10.0, label = proctimes$V2) + 
+	scale_colour_manual(values=c("#FF7777", "#0066CC")) + 
+	theme_bw() + 
+	guides(fill=guide_legend(title=NULL)) + 
+	theme(legend.title=element_blank())
+
+ggsave("usage_disk.png", pplot, width=14)

caja/playground/monitor/run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+CPU_FILE_PATH=$1
+MEM_FILE_PATH=$2
+DISK_FILE_PATH=$3
+PROCTIMES_FILE_PATH=$4
+
+RESULT_DISK_FILE_PATH=$(pwd)/global_disk_usage.csv
+
+echo "Filtering and summing disk usage data..."
+
+python filter-sum-disk-usage.py $DISK_FILE_PATH $RESULT_DISK_FILE_PATH
+
+echo "Generating graphics..."
+
+Rscript plot-usages.r $CPU_FILE_PATH $MEM_FILE_PATH $RESULT_DISK_FILE_PATH $PROCTIMES_FILE_PATH 

caja/playground/run

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+rm monitor/procs
+
+./monitor/collect-cpu-usage.sh >monitor/cpu-usage &
+./monitor/collect-disk-usage.sh >monitor/disk-usage &
+./monitor/collect-memory-usage.sh >monitor/memory-usage &
+
+cd src
+
+./run
+
+cd ..
+
+ps -ef | grep ./collect | awk '{print $2}' | xargs -n 1 -P 4 kill -9
+
+cd monitor
+
+(cat procs | grep "end") > filter-procs
+
+Rscript plot-usages.r cpu-usage memory-usage disk-usage filter-procs

caja/playground/src/criteria

@@ -0,0 +1 @@
+AEMMF.tif 1 1

caja/playground/src/merge.cpp

@@ -0,0 +1,66 @@
+#include "tiffio.h"
+#include <bits/stdc++.h>
+#include <time.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+using namespace std;
+
+void logger(string description){
+    timespec res;
+    clock_gettime(CLOCK_REALTIME, &res);
+    cout << res.tv_sec << " " << description.c_str() << " " << getpid() << endl;
+}
+
+// ./run filename1 filename2 filename3 filename4
+int main(int argc, char* argv[]){
+    logger("merge start");
+    TIFF* tifs[argc - 1];
+
+    for(int i = 1; i < argc; i++){
+        cerr << "open " << string(argv[i]) << endl;
+        tifs[i - 1] = TIFFOpen(argv[i], "rm");
+    }
+
+    int ntiffs = argc - 1;
+    int width, height, sampleperpixel;
+
+    TIFFGetField(tifs[0], TIFFTAG_IMAGEWIDTH, &width);
+    TIFFGetField(tifs[0], TIFFTAG_IMAGELENGTH, &height);
+    TIFFGetField(tifs[0], TIFFTAG_SAMPLESPERPIXEL, &sampleperpixel);
+
+    TIFF *out = TIFFOpen("merged.tif", "w8m");
+    TIFFSetField(out, TIFFTAG_IMAGEWIDTH     , width); 
+    TIFFSetField(out, TIFFTAG_IMAGELENGTH    , height);
+    TIFFSetField(out, TIFFTAG_BITSPERSAMPLE  , 64);
+    TIFFSetField(out, TIFFTAG_SAMPLEFORMAT  , 3);
+    TIFFSetField(out, TIFFTAG_COMPRESSION    , 1);
+    TIFFSetField(out, TIFFTAG_PHOTOMETRIC    , 1);
+    TIFFSetField(out, TIFFTAG_ORIENTATION    , 1);
+    TIFFSetField(out, TIFFTAG_SAMPLESPERPIXEL, 1);
+    TIFFSetField(out, TIFFTAG_ROWSPERSTRIP   , 8);
+    TIFFSetField(out, TIFFTAG_RESOLUTIONUNIT , 1);
+    TIFFSetField(out, TIFFTAG_XRESOLUTION    , 1);
+    TIFFSetField(out, TIFFTAG_YRESOLUTION    , 1);
+    TIFFSetField(out, TIFFTAG_PLANARCONFIG   , PLANARCONFIG_CONTIG   );
+
+    double* data = new double[width];
+    double* tmp = new double[width];
+
+    for(int i = 0; i < height; i++){
+        for(int j = 0; j < width; j++) data[j] = 0;
+
+        for(int k = 0; k < ntiffs; k++){
+            TIFFReadScanline(tifs[k], tmp, i);
+            for(int j = 0; j < width; j++) data[j] += tmp[j];
+        }
+
+        TIFFWriteScanline(out, data, i);
+    }
+
+    logger("merge end");
+    TIFFClose(out);
+
+    for(int i = 0; i < ntiffs; i++) TIFFClose(tifs[i]);
+    return 0;
+}

caja/playground/src/run

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Compiling programs, change to makefile after.
+#g++ -std=c++14 merge.cpp -o merge -ltiff
+#g++ -std=c++14 umbu.cpp -o umbu -ltiff
+
+# Run criteria in parallel
+cat criteria | xargs -n 3 -P 8 ./umbu >>../monitor/procs
+# criteria should follow this pattern
+# name_tif.tif weight p_parameter
+
+input=""
+
+for f in out.*.tif; do
+  input=$input" "$f
+done
+
+# Sum all flow results
+./merge$input >> ../monitor/procs