Merge pull request #195 from fogbow/develop

Fixing garbageCollector. Refactoring execBenchmark.

bin/fogbow-create-reverse-tunnel

@@ -23,3 +23,9 @@ autossh &
 EOL
 chmod +x /bin/fogbow-autossh
 setsid /bin/fogbow-autossh
+cat > /bin/create-fogbow-tunnel << EOL
+#!/bin/sh
+REMOTE_PORT=\$(curl -X POST #TOKEN_HOST#:#TOKEN_HOST_HTTP_PORT#/token/#TOKEN_ID#-\${1})
+ssh $SSH_OPTIONS -f -N -R 0.0.0.0:\$REMOTE_PORT:localhost:\${2} #TOKEN_ID#-\${1}@#TOKEN_HOST# -p #TOKEN_HOST_SSH_PORT#
+EOL
+chmod +x /bin/create-fogbow-tunnel

examples/lvl-user-data.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+create-fogbow-tunnel http 80
+sed -i 's/include \/etc\/nginx\/naxsi/#include \/etc\/nginx/g' /etc/nginx/sites-enabled/default
+service nginx restart

manager.conf.example

@@ -6,132 +6,145 @@ xmpp_port=5347
 rendezvous_jid=rendezvous.test.com
 greensitter_jid=greensitter.test.com
 
-# That flavor specification will be transformed into requirements.
-flavor_fogbow_small={mem=512, cpu=1}
-flavor_fogbow_medium={mem=1024, cpu=2}
-flavor_fogbow_large={mem=2048, cpu=4}
-
-# Compute Opennebula
-compute_class=org.fogbowcloud.manager.core.plugins.opennebula.OpenNebulaComputePlugin
-compute_one_url=http://localhost:2633/RPC2
-compute_one_network_id=1
-
-# Setting used to find the correct flavor according to informed templates.
-# types :all or specific templates(template1,template2) 
-compute_one_templates=all
-
-# Settings used by ONE compute plugin to register new images in the cloud
-# ID of datastore to register the image
-compute_one_datastore_id=1
-# To register a new image, the image file needs to be in the some machine where ONE is running
-# If the fogbow manager is running in a different machine, set the SSH properties to transfer the image
-# Or if the fogbow manager is in the same machine, leave it blank
-compute_one_ssh_host=127.0.0.1
-compute_one_ssh_port=22
-compute_one_ssh_username=fogbow
-# The SSH try to access using private key, set the path to ssh id_rsa file
-compute_one_ssh_key_file=/home/fogbow/.ssh/id_rsa
-# Set the directory to copy images in remote host
-compute_one_ssh_target_temp_folder=/tmp/images
-
-# Compute Openstack OCCI
-compute_class=org.fogbowcloud.manager.core.plugins.openstack.OpenStackOCCIComputePlugin
-compute_openstack_v2api_url=http://localhost:8182
-compute_occi_url=http://localhost:8182
-compute_occi_os_scheme=http://schemas.openstack.org/template/os#
-compute_occi_instance_scheme=http://schemas.openstack.org/compute/instance#
-compute_occi_resource_scheme=http://schemas.openstack.org/template/resource#
-compute_occi_network_id=ea51ed0c-0e8a-448d-8202-c79777109ffe
-
-# Compute Opennebula OCCI
-compute_class=org.fogbowcloud.manager.core.plugins.opennebula.OpenNebulaOCCIComputePlugin
-compute_one_url=http://localhost:2633/RPC2
-compute_occi_url=http://localhost:3000
-compute_occi_template_scheme=http://occi.localhost/occi/infrastructure/os_tpl#
-compute_occi_resource_scheme=http://schema.fedcloud.egi.eu/occi/infrastructure/resource_tpl#
-compute_occi_flavors_small={cpu=1,mem=512}
-compute_occi_flavors_medium={cpu=2,mem=1024}
-compute_occi_flavors_large={cpu=4,mem=2048}
-
-# If you are using the EgiImageStoragePlugin
-image_storage_class=org.fogbowcloud.manager.core.plugins.imagestorage.egi.EgiImageStoragePlugin
-image_storage_egi_base_url=http://lsd.ufcg.edu.br/~user/vm/ 
-image_storage_egi_tmp_storage=/tmp/ 
-
-# If you are using the EgiApplianceImageStoragePlugin
-image_storage_class=org.fogbowcloud.manager.core.plugins.imagestorage.appliance.EgiApplianceImageStoragePlugin
-image_storage_appliance_base_url=http://appliance-repo.egi.eu/images
-image_storage_appliance_tmp_storage=/tmp/
-
-# If you are using the VMCatcherStoragePlugin
-image_storage_class=org.fogbowcloud.manager.core.plugins.imagestorage.vmcatcher.VMCatcherStoragePlugin
-image_storage_vmcatcher_glancepush_vmcmapping_file=/etc/vmcatcher/vmcmapping
-image_storage_vmcatcher_push_method=glancepush
-image_storage_vmcatcher_push_method=cesga
-
-# Applies to all image storage plugins
+## Static mapping from flavors to requirements (Optional)
+# flavor_fogbow_small={mem=512, cpu=1}
+# flavor_fogbow_medium={mem=1024, cpu=2}
+# flavor_fogbow_large={mem=2048, cpu=4}
+
+## Compute Plugin (Openstack)
+compute_class=org.fogbowcloud.manager.core.plugins.compute.openstack.OpenStackNovaV2ComputePlugin
+compute_novav2_url=http://localhost:8774
+compute_glancev2_url=http://localhost:9292
+compute_glancev2_image_visibility=private
+compute_novav2_network_id=ea51ed0c-0e8a-448d-8202-c79777109ffe
+
+## Compute Plugin (Opennebula)
+# compute_class=org.fogbowcloud.manager.core.plugins.compute.opennebula.OpenNebulaComputePlugin
+# compute_one_url=http://localhost:2633/RPC2
+# compute_one_network_id=1 
+# compute_one_templates=all
+# compute_one_datastore_id=1
+# compute_one_ssh_host=127.0.0.1
+# compute_one_ssh_port=22
+# compute_one_ssh_username=fogbow
+# compute_one_ssh_key_file=/home/fogbow/.ssh/id_rsa
+# compute_one_ssh_target_temp_folder=/tmp/images
+
+## Compute Plugin (Openstack OCCI)
+# compute_class=org.fogbowcloud.manager.core.plugins.compute.openstack.OpenStackOCCIComputePlugin
+# compute_openstack_v2api_url=http://localhost:8182
+# compute_occi_url=http://localhost:8182
+# compute_occi_os_scheme=http://schemas.openstack.org/template/os#
+# compute_occi_instance_scheme=http://schemas.openstack.org/compute/instance#
+# compute_occi_resource_scheme=http://schemas.openstack.org/template/resource#
+# compute_occi_network_id=ea51ed0c-0e8a-448d-8202-c79777109ffe
+
+## Compute Plugin (Opennebula OCCI)
+# compute_class=org.fogbowcloud.manager.core.plugins.compute.opennebula.OpenNebulaOCCIComputePlugin
+# compute_one_url=http://localhost:2633/RPC2
+# compute_occi_url=http://localhost:3000
+# compute_occi_template_scheme=http://occi.localhost/occi/infrastructure/os_tpl#
+# compute_occi_resource_scheme=http://schema.fedcloud.egi.eu/occi/infrastructure/resource_tpl#
+# compute_occi_flavors_small={cpu=1,mem=512}
+# compute_occi_flavors_medium={cpu=2,mem=1024}
+# compute_occi_flavors_large={cpu=4,mem=2048}
+
+## Image Storage Plugin (HTTPDownload)
+image_storage_class=org.fogbowcloud.manager.core.plugins.imagestorage.http.HTTPDownloadImageStoragePlugin
+image_storage_http_base_url=http://appliance-repo.egi.eu/images
+image_storage_http_tmp_storage=/tmp/
+
+## Image Storage Plugin (VMCatcher)
+# image_storage_class=org.fogbowcloud.manager.core.plugins.imagestorage.vmcatcher.VMCatcherStoragePlugin
+# image_storage_vmcatcher_use_sudo=false
+# image_storage_vmcatcher_env_VMCATCHER_RDBMS="sqlite:////var/lib/vmcatcher/vmcatcher.db"
+# image_storage_vmcatcher_env_VMCATCHER_CACHE_DIR_CACHE="/var/lib/vmcatcher/cache"
+# image_storage_vmcatcher_env_VMCATCHER_CACHE_DIR_DOWNLOAD="/var/lib/vmcatcher/cache/partial"
+# image_storage_vmcatcher_env_VMCATCHER_CACHE_DIR_EXPIRE="/var/lib/vmcatcher/cache/expired"
+## glancepush specific
+# image_storage_vmcatcher_push_method=glancepush
+# image_storage_vmcatcher_glancepush_vmcmapping_file=/etc/vmcatcher/vmcmapping
+# image_storage_vmcatcher_env_VMCATCHER_CACHE_EVENT="python /var/lib/vmcatcher/gpvcmupdate.py"
+## one specific
+# image_storage_vmcatcher_push_method=cesga
+# image_storage_vmcatcher_env_VMCATCHER_CACHE_EVENT="python /var/lib/vmcatcher/vmcatcher_eventHndl_ON"
+# image_storage_vmcatcher_env_ONE_AUTH="/etc/vmcatcher/one_auth"
+
+## Static mapping between local image ids and image names
+## Applies to all image storage plugins
 image_storage_static_fogbow-linux-x86=55d938ef-57d1-44ea-8155-6036d170780a 
 image_storage_static_fogbow-ubuntu-1204=81765250-a4e4-440d-a215-43c9c0849120
 
-member_picker_class=org.fogbowcloud.manager.core.RoundRobinMemberPicker
-#If you are using NoFMemberPicker or NoFPrioritizationPlugin classes
-nof_trustworthy=false
-nof_prioritize_local=true
+## Member Picker Plugin (Round Robin) 
+member_picker_class=org.fogbowcloud.manager.core.plugins.memberpicker.RoundRobinMemberPickerPlugin
 
-local_prioritization_plugin_class=org.fogbowcloud.manager.core.plugins.prioritization.FCFSPrioritizationPlugin
+## Member Picker Plugin (Nof Member Picker)
+# member_picker_class=org.fogbowcloud.manager.core.plugins.memberpicker.NoFMemberPickerPlugin
+
+## Local Prioritization Plugin
+local_prioritization_plugin_class=org.fogbowcloud.manager.core.plugins.prioritization.fcfs.FCFSPrioritizationPlugin
+## Remote Prioritization Plugin
 remote_prioritization_plugin_class=org.fogbowcloud.manager.core.plugins.prioritization.nof.NoFPrioritizationPlugin
+nof_prioritize_local=true
 
-# Compute Openstack
-compute_class=org.fogbowcloud.manager.core.plugins.openstack.OpenStackNovaV2ComputePlugin
-compute_novav2_url=http://localhost:8774
-compute_glancev2_url=http://localhost:9292
-compute_novav2_network_id=ea51ed0c-0e8a-448d-8202-c79777109ffe
+## If you are using NoFMemberPicker or NoFPrioritizationPlugin classes
+nof_trustworthy=false
 
-# Local identity
-local_identity_class=org.fogbowcloud.manager.core.plugins.openstack.KeystoneIdentityPlugin
+## Local identity (Keystone Identity Plugin)
+local_identity_class=org.fogbowcloud.manager.core.plugins.identity.openstack.KeystoneIdentityPlugin
 local_identity_url=http://localhost:5000
-
-# Federation identity
-federation_authorization_class=org.fogbowcloud.manager.core.plugins.common.AllowAllAuthorizationPlugin
-federation_identity_class=org.fogbowcloud.manager.core.plugins.openstack.KeystoneIdentityPlugin
-federation_identity_url=http://localhost:5000
-
 local_proxy_account_user_name=fogbow
 local_proxy_account_password=fogbow
 local_proxy_account_tenant_name=demo
 
-member_validator=org.fogbowcloud.manager.core.DefaultMemberValidator
-member_validator_ca_dir=
+## Local Identity (X509 Identity Plugin)
+# local_identity_class=org.fogbowcloud.manager.core.plugins.x509.X509IdentityPlugin
+# x509_ca_dir_path=/path/to/ca/directory
 
-cert_path=
+## Local Identity (Opennebula Identity Plugin)
+# local_identity_class=org.fogbowcloud.manager.core.plugins.opennebula.OpenNebulaIdentityPlugin
+# local_identity_url=http://localhost:2633/RPC2
 
-scheduler_period=30000
-token_update_period=300000
-instance_monitoring_period=120000
-served_request_monitoring_period=120000
-garbage_collector_period=240000
-asyn_request_waiting_interval=300000
+## Authorizatio Plugin
+federation_authorization_class=org.fogbowcloud.manager.core.plugins.authorization.AllowAllAuthorizationPlugin
 
-ssh_tunnel_public_host=150.165.80.1
-ssh_tunnel_private_host=10.0.0.1
-ssh_tunnel_host_http_port=2223
-ssh_tunnel_host_port=2222
+## Federation identity (Keystone Identity Plugin)
+federation_identity_class=org.fogbowcloud.manager.core.plugins.identity.openstack.KeystoneIdentityPlugin
+federation_identity_url=http://localhost:5000
 
-my_ip=127.0.0.1
+## Federation Identity (Opennebula Identity Plugin)
+# federation_identity_class=org.fogbowcloud.manager.core.plugins.opennebula.OpenNebulaIdentityPlugin
+# federation_identity_url=http://localhost:2633/RPC2
 
-http_port=8182
+## Federation Identity (VOMS Identity PLugin)
+# federation_identity_class=org.fogbowcloud.manager.core.plugins.identity.voms.VomsIdentityPlugin
+# path_vomses=/etc/vomses
+# path_trust_anchors=/etc/grid-security/certificates
+# path_vomsdir=/etc/grid-security/vomsdir
 
-max_whoisalive_manager_count=100
+## Member Validator
+member_validator_class=org.fogbowcloud.manager.core.plugins.memberauthorization.DefaultMemberAuthorizationPlugin
+member_validator_ca_dir=
+
+token_host_public_address=150.165.80.1
+token_host_private_address=10.0.0.1
+token_host_http_port=2223
+token_host_port=2222
+
+my_ip=127.0.0.1
 
+## Accounting Plugin
 accounting_class=org.fogbowcloud.manager.core.plugins.accounting.FCUAccountingPlugin
 accounting_update_period=300000
 accounting_datastore_url=jdbc:h2:/tmp/usage
 
-# SSH Benchmarking plugin
+## Benchmarking (SSH Benchmarking plugin)
 benchmarking_class=org.fogbowcloud.manager.core.plugins.benchmarking.SSHBenchmarkingPlugin
-# Benchmarking script to use with SSH Benchmarking plugin
+## Benchmarking script to use with SSH Benchmarking plugin
 ssh_benchmarking_script_url=http://downloads.fogbowcloud.org/benchmark/script_ssh_benchmarking.sh
-
-# Manager public and private keys
+## Manager public and private keys
 ssh_private_key=/etc/fogbow-manager/ssh/id_rsa
-ssh_public_key=/etc/fogbow-manager/ssh/id_rsa.pub
+ssh_public_key=/etc/fogbow-manager/ssh/id_rsa.pub
+
+## Benchmarking (Vanilla Benchmarking Plugin)
+# benchmarking_class=org.fogbowcloud.manager.core.plugins.benchmarking.VanillaBenchmarkingPlugin

pom.xml

@@ -166,10 +166,10 @@
       <version>2.2.0</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpclient</artifactId>
-      <version>4.0-alpha4</version>
-    </dependency>
+		<groupId>org.apache.httpcomponents</groupId>
+		<artifactId>httpclient</artifactId>
+		<version>4.4</version>
+	</dependency>
     <dependency>
       <groupId>org.json</groupId>
       <artifactId>json</artifactId>

src/main/java/org/fogbowcloud/manager/Main.java

@@ -6,24 +6,24 @@
 import org.apache.log4j.ConsoleAppender;
 import org.apache.log4j.Logger;
 import org.fogbowcloud.manager.core.ConfigurationConstants;
-import org.fogbowcloud.manager.core.DefaultMemberValidator;
-import org.fogbowcloud.manager.core.FederationMemberPicker;
-import org.fogbowcloud.manager.core.FederationMemberValidator;
 import org.fogbowcloud.manager.core.ManagerController;
-import org.fogbowcloud.manager.core.RoundRobinMemberPicker;
 import org.fogbowcloud.manager.core.plugins.AccountingPlugin;
 import org.fogbowcloud.manager.core.plugins.AuthorizationPlugin;
 import org.fogbowcloud.manager.core.plugins.BenchmarkingPlugin;
 import org.fogbowcloud.manager.core.plugins.ComputePlugin;
+import org.fogbowcloud.manager.core.plugins.FederationMemberAuthorizationPlugin;
+import org.fogbowcloud.manager.core.plugins.FederationMemberPickerPlugin;
 import org.fogbowcloud.manager.core.plugins.IdentityPlugin;
 import org.fogbowcloud.manager.core.plugins.ImageStoragePlugin;
 import org.fogbowcloud.manager.core.plugins.PrioritizationPlugin;
 import org.fogbowcloud.manager.core.plugins.accounting.FCUAccountingPlugin;
 import org.fogbowcloud.manager.core.plugins.benchmarking.VanillaBenchmarkingPlugin;
-import org.fogbowcloud.manager.core.plugins.imagestorage.egi.EgiImageStoragePlugin;
+import org.fogbowcloud.manager.core.plugins.imagestorage.http.HTTPDownloadImageStoragePlugin;
+import org.fogbowcloud.manager.core.plugins.memberauthorization.DefaultMemberAuthorizationPlugin;
+import org.fogbowcloud.manager.core.plugins.memberpicker.RoundRobinMemberPickerPlugin;
 import org.fogbowcloud.manager.core.plugins.prioritization.TwoFoldPrioritizationPlugin;
 import org.fogbowcloud.manager.occi.OCCIApplication;
-import org.fogbowcloud.manager.occi.core.ResourceRepository;
+import org.fogbowcloud.manager.occi.model.ResourceRepository;
 import org.fogbowcloud.manager.xmpp.ManagerXmppComponent;
 import org.restlet.Component;
 import org.restlet.data.Protocol;
@@ -35,6 +35,7 @@ public class Main {
 
 	private static final Logger LOGGER = Logger.getLogger(Main.class);
 	private static final int EXIT_ERROR_CODE = 128;
+	private static final int DEFAULT_HTTP_PORT = 8182;
 
 	public static void main(String[] args) throws Exception {
 		configureLog4j();
@@ -80,10 +81,10 @@ public static void main(String[] args) throws Exception {
 			System.exit(EXIT_ERROR_CODE);
 		}
 
-		FederationMemberValidator validator = new DefaultMemberValidator(properties);
+		FederationMemberAuthorizationPlugin validator = new DefaultMemberAuthorizationPlugin(properties);
 		try {
-			validator = (FederationMemberValidator) createInstance(
-					ConfigurationConstants.MEMBER_VALIDATOR_KEY, properties);
+			validator = (FederationMemberAuthorizationPlugin) createInstance(
+					ConfigurationConstants.MEMBER_VALIDATOR_CLASS_KEY, properties);
 		} catch (Exception e) {
 			LOGGER.warn("Member Validator not especified in the properties.");
 			System.exit(EXIT_ERROR_CODE);
@@ -100,7 +101,7 @@ public static void main(String[] args) throws Exception {
 			imageStoragePlugin = (ImageStoragePlugin) createInstanceWithComputePlugin(
 					ConfigurationConstants.IMAGE_STORAGE_PLUGIN_CLASS, properties, computePlugin);
 		} catch (Exception e) {
-			imageStoragePlugin = new EgiImageStoragePlugin(properties, computePlugin);
+			imageStoragePlugin = new HTTPDownloadImageStoragePlugin(properties, computePlugin);
 			LOGGER.warn("Image Storage plugin not specified in properties. Using the default one.", e);
 		}
 
@@ -122,13 +123,13 @@ public static void main(String[] args) throws Exception {
 			LOGGER.warn("Accounting plugin not specified in properties. Using the default one.", e);
 		}
 
-		FederationMemberPicker memberPickerPlugin = null;
+		FederationMemberPickerPlugin memberPickerPlugin = null;
 		try {
-			memberPickerPlugin = (FederationMemberPicker) createInstanceWithAccoutingPlugin(
+			memberPickerPlugin = (FederationMemberPickerPlugin) createInstanceWithAccoutingPlugin(
 					ConfigurationConstants.MEMBER_PICKER_PLUGIN_CLASS_KEY, properties,
 					accountingPlugin);
 		} catch (Exception e) {
-			memberPickerPlugin = new RoundRobinMemberPicker(properties, accountingPlugin);
+			memberPickerPlugin = new RoundRobinMemberPickerPlugin(properties, accountingPlugin);
 			LOGGER.warn("Member picker plugin not specified in properties. Using the default one.", e);
 		}
 
@@ -171,8 +172,8 @@ public static void main(String[] args) throws Exception {
 
 		try {
 			Component http = new Component();
-			http.getServers().add(Protocol.HTTP,
-					Integer.parseInt(properties.getProperty(ConfigurationConstants.HTTP_PORT_KEY)));
+			String httpPort = properties.getProperty(ConfigurationConstants.HTTP_PORT_KEY);			
+			http.getServers().add(Protocol.HTTP, httpPort == null ? DEFAULT_HTTP_PORT : Integer.parseInt(httpPort));
 			http.getDefaultHost().attach(application);
 			http.start();
 		} catch (Exception e) {