Merge pull request #397 from Temikus/parallelize_more

More parallelization
fog · Jul 18, 2018 · 8780575 · 8780575
2 parents d1d8058 + 306f2af
commit 8780575
Show file tree

Hide file tree

Showing 52 changed files with 258 additions and 75 deletions.
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -16,6 +16,9 @@ Style/RegexpLiteral:
 Style/FrozenStringLiteralComment:
   Enabled: false
 
+Style/MutableConstant:
+  Enabled: false
+
 Style/HashSyntax:
   EnforcedStyle: no_mixed_keys
 
@@ -30,7 +33,7 @@ AllCops:
     - "vendor/**/*"
     - "db/schema.rb"
   UseCache: false
-  TargetRubyVersion: 2.1
+  TargetRubyVersion: 2.3
 Style/CollectionMethods:
   Description: Preferred collection methods.
   StyleGuide: https://github.com/bbatsov/ruby-style-guide#map-find-select-reduce-size

diff --git a/README.md b/README.md
@@ -4,11 +4,13 @@
 
 The main maintainers for the Google sections are @icco, @Temikus and @plribeiro3000. Please send pull requests to them.
 
-**As of v0.1.1, Google no longer supports Ruby versions less than 2.0.0.**
+## Important notices
 
-**As of v1.0.0, fog-google includes google-api-client as a dependency**
+ - As of **v0.1.1**, Google no longer supports Ruby versions less than 2.0.0.  
 
-**See [MIGRATING.md](MIGRATING.md) for migration between major versions**
+ - As of **v1.0.0**, fog-google includes google-api-client as a dependency, there is no need to include it separately anymore.
+
+See **[MIGRATING.md](MIGRATING.md)** for migration between major versions.
 
 ## Storage
 

diff --git a/ci/docker-image/Dockerfile b/ci/docker-image/Dockerfile
@@ -13,8 +13,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y -qq update && apt-get -y -qq insta
   libxslt-dev
 
 # Ubuntu 16.04 will fetch us 2.3.x without any issues.
-RUN DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ruby ruby-dev
-
-RUN apt-get clean
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ruby ruby-dev && apt-get clean
 
 RUN gem install bundler
diff --git a/ci/pipeline.yml b/ci/pipeline.yml
@@ -1,5 +1,5 @@
 jobs:
-  - name: integration-tests-compute
+  - name: integration-compute-core_compute
     serial: true
     plan:
       - get: fog-google-src
@@ -9,10 +9,10 @@ jobs:
       - put: pull-request
         params: {path: fog-google-src, status: pending}
 
-      - task: full-integration-tests
+      - task: run-integration-tests
         file: fog-google-src/ci/tasks/run-int.yml
         params:
-          rake_task: test:compute
+          rake_task: test:compute-core_compute
           codecov_token: {{codecov_token}}
           google_project: {{google_project}}
           google_json_key_data: {{google_json_key_data}}
@@ -22,7 +22,79 @@ jobs:
           params:
             path: fog-google-src
             status: failure
-
+
+  - name: integration-compute-core_networking
+    serial: true
+    plan:
+      - get: fog-google-src
+        resource: pull-request
+        version: every
+        trigger: true
+      - put: pull-request
+        params: {path: fog-google-src, status: pending}
+
+      - task: run-integration-tests
+        file: fog-google-src/ci/tasks/run-int.yml
+        params:
+          rake_task: test:compute-core_networking
+          codecov_token: {{codecov_token}}
+          google_project: {{google_project}}
+          google_json_key_data: {{google_json_key_data}}
+          google_client_email: {{google_client_email}}
+        on_failure:
+          put: pull-request
+          params:
+            path: fog-google-src
+            status: failure
+
+  - name: integration-compute-instance_groups
+    serial: true
+    plan:
+      - get: fog-google-src
+        resource: pull-request
+        version: every
+        trigger: true
+      - put: pull-request
+        params: {path: fog-google-src, status: pending}
+
+      - task: run-integration-tests
+        file: fog-google-src/ci/tasks/run-int.yml
+        params:
+          rake_task: test:compute-instance_groups
+          codecov_token: {{codecov_token}}
+          google_project: {{google_project}}
+          google_json_key_data: {{google_json_key_data}}
+          google_client_email: {{google_client_email}}
+        on_failure:
+          put: pull-request
+          params:
+            path: fog-google-src
+            status: failure
+
+  - name: integration-compute-loadbalancing
+    serial: true
+    plan:
+      - get: fog-google-src
+        resource: pull-request
+        version: every
+        trigger: true
+      - put: pull-request
+        params: {path: fog-google-src, status: pending}
+
+      - task: run-integration-tests
+        file: fog-google-src/ci/tasks/run-int.yml
+        params:
+          rake_task: test:compute-loadbalancing
+          codecov_token: {{codecov_token}}
+          google_project: {{google_project}}
+          google_json_key_data: {{google_json_key_data}}
+          google_client_email: {{google_client_email}}
+        on_failure:
+          put: pull-request
+          params:
+            path: fog-google-src
+            status: failure
+
   - name: integration-tests-monitoring
     serial: true
     plan:
@@ -33,7 +105,7 @@ jobs:
       - put: pull-request
         params: {path: fog-google-src, status: pending}
 
-      - task: full-integration-tests
+      - task: run-integration-tests
         file: fog-google-src/ci/tasks/run-int.yml
         params:
           rake_task: test:monitoring
@@ -57,7 +129,7 @@ jobs:
       - put: pull-request
         params: {path: fog-google-src, status: pending}
 
-      - task: full-integration-tests
+      - task: run-integration-tests
         file: fog-google-src/ci/tasks/run-int.yml
         params:
           rake_task: test:pubsub
@@ -71,7 +143,31 @@ jobs:
             path: fog-google-src
             status: failure
 
-  - name: integration-tests-sql
+  - name: integration-sqlv1
+    serial: true
+    plan:
+      - get: fog-google-src
+        resource: pull-request
+        version: every
+        trigger: true
+      - put: pull-request
+        params: {path: fog-google-src, status: pending}
+
+      - task: run-integration-tests
+        file: fog-google-src/ci/tasks/run-int.yml
+        params:
+          rake_task: test:sql-sqlv1
+          codecov_token: {{codecov_token}}
+          google_project: {{google_project}}
+          google_json_key_data: {{google_json_key_data}}
+          google_client_email: {{google_client_email}}
+        on_failure:
+          put: pull-request
+          params:
+            path: fog-google-src
+            status: failure
+
+  - name: integration-sqlv2
     serial: true
     plan:
       - get: fog-google-src
@@ -81,10 +177,10 @@ jobs:
       - put: pull-request
         params: {path: fog-google-src, status: pending}
 
-      - task: full-integration-tests
+      - task: run-integration-tests
         file: fog-google-src/ci/tasks/run-int.yml
         params:
-          rake_task: test:sql
+          rake_task: test:sql-sqlv2
           codecov_token: {{codecov_token}}
           google_project: {{google_project}}
           google_json_key_data: {{google_json_key_data}}
@@ -105,7 +201,7 @@ jobs:
       - put: pull-request
         params: {path: fog-google-src, status: pending}
 
-      - task: full-integration-tests
+      - task: run-integration-tests
         file: fog-google-src/ci/tasks/run-int.yml
         params:
           rake_task: test:storage
@@ -124,10 +220,14 @@ jobs:
       - get: fog-google-src
         resource: pull-request
         passed: [integration-tests-storage,
-                 integration-tests-sql,
                  integration-tests-pubsub,
                  integration-tests-monitoring,
-                 integration-tests-compute]
+                 integration-sqlv1,
+                 integration-sqlv2,
+                 integration-compute-core_compute,
+                 integration-compute-core_networking,
+                 integration-compute-instance_groups,
+                 integration-compute-loadbalancing]
         trigger: true
         on_success:
           put: pull-request

diff --git a/ci/tasks/run-int.sh b/ci/tasks/run-int.sh
@@ -29,7 +29,7 @@ EOL
 
 pushd ${release_dir} > /dev/null
 
-bundle install
+bundle install --jobs=3 --retry=3
 
 FOG_MOCK=false COVERAGE=true CODECOV_TOKEN=${codecov_token} rake ${rake_task}
 

diff --git a/ci/tasks/run-int.yml b/ci/tasks/run-int.yml
@@ -2,7 +2,7 @@
 platform: linux
 image_resource:
   type: docker-image
-  source: {repository: everlag/fog-google}
+  source: {repository: rubyfog/fog-google}
 inputs:
 - name: fog-google-src
   path: src/fog-google

diff --git a/tasks/test.rake b/tasks/test.rake
@@ -24,20 +24,33 @@ namespace :test do
     t.name = "unit"
     t.description = "Run Unit tests"
     t.libs << "test"
-    t.pattern = FileList['test/unit/**/test_*.rb']
+    t.pattern = FileList["test/unit/**/test_*.rb"]
     t.warning = false
     t.verbose = true
   end
 
-  Rake::TestTask.new do |t|
-    t.name = "compute"
-    t.description = "Run Compute API tests"
-    t.libs << "test"
-    t.pattern = FileList["test/integration/compute/test_*.rb"]
-    t.warning = false
-    t.verbose = true
+  # This autogenerates rake tasks based on test folder structures
+  # This is done to simplify running many test suites in parallel
+  COMPUTE_TEST_TASKS = []
+  Dir.glob("test/integration/compute/**").each do |task|
+    suite_collection = task.gsub(/test\/integration\/compute\//, "")
+    component_name = task.gsub(/test\/integration\//, "").split("/").first
+    Rake::TestTask.new(:"#{component_name}-#{suite_collection}") do |t|
+      t.libs << "test"
+      t.description = "Autotask - run #{component_name} integration tests - #{suite_collection}"
+      t.pattern = FileList["test/integration/#{component_name}/#{suite_collection}/test_*.rb"]
+      t.warning = false
+      t.verbose = true
+    end
+    COMPUTE_TEST_TASKS << "#{component_name}-#{suite_collection}"
   end
 
+  desc "Run Compute API tests"
+  task :compute => COMPUTE_TEST_TASKS
+
+  desc "Run Compute API tests in parallel"
+  multitask :compute_parallel => COMPUTE_TEST_TASKS
+
   Rake::TestTask.new do |t|
     t.name = "monitoring"
     t.description = "Run Monitoring API tests"
@@ -56,6 +69,28 @@ namespace :test do
     t.verbose = true
   end
 
+  # This autogenerates rake tasks based on test folder structures
+  # This is done to simplify running many test suites in parallel
+  SQL_TEST_TASKS = []
+  Dir.glob("test/integration/sql/**").each do |task|
+    suite_collection = task.gsub(/test\/integration\/sql\//, "")
+    component_name = task.gsub(/test\/integration\//, "").split("/").first
+    Rake::TestTask.new(:"#{component_name}-#{suite_collection}") do |t|
+      t.libs << "test"
+      t.description = "Autotask - run #{component_name} integration tests - #{suite_collection}"
+      t.pattern = FileList["test/integration/#{component_name}/#{suite_collection}/test_*.rb"]
+      t.warning = false
+      t.verbose = true
+    end
+    SQL_TEST_TASKS << "#{component_name}-#{suite_collection}"
+  end
+
+  desc "Run SQL API tests"
+  task :compute => SQL_TEST_TASKS
+
+  desc "Run SQL API tests in parallel"
+  multitask :compute_parallel => SQL_TEST_TASKS
+
   Rake::TestTask.new do |t|
     t.name = "sql"
     t.description = "Run SQL API tests"

diff --git a/test/integration/compute/core_compute/test_coverage.rb b/test/integration/compute/core_compute/test_coverage.rb
@@ -0,0 +1,6 @@
+require "helpers/integration_test_helper"
+
+# This is a simple coverage helper that helps differentiate
+# the tests when run in parallel so the final coverage report
+# can be properly combined together from multiple runners
+SimpleCov.command_name "test:compute-core_compute" if ENV["COVERAGE"]
diff --git a/test/integration/compute/test_disk_types.rb → ...n/compute/core_compute/test_disk_types.rb b/test/integration/compute/test_disk_types.rb → ...n/compute/core_compute/test_disk_types.rb
diff --git a/test/integration/compute/test_disks.rb → ...ration/compute/core_compute/test_disks.rb b/test/integration/compute/test_disks.rb → ...ration/compute/core_compute/test_disks.rb
diff --git a/test/integration/compute/test_images.rb → ...ation/compute/core_compute/test_images.rb b/test/integration/compute/test_images.rb → ...ation/compute/core_compute/test_images.rb
diff --git a/...integration/compute/test_machine_types.rb → ...ompute/core_compute/test_machine_types.rb b/...integration/compute/test_machine_types.rb → ...ompute/core_compute/test_machine_types.rb
diff --git a/test/integration/compute/test_operations.rb → ...n/compute/core_compute/test_operations.rb b/test/integration/compute/test_operations.rb → ...n/compute/core_compute/test_operations.rb
diff --git a/test/integration/compute/test_projects.rb → ...ion/compute/core_compute/test_projects.rb b/test/integration/compute/test_projects.rb → ...ion/compute/core_compute/test_projects.rb
diff --git a/test/integration/compute/test_regions.rb → ...tion/compute/core_compute/test_regions.rb b/test/integration/compute/test_regions.rb → ...tion/compute/core_compute/test_regions.rb
diff --git a/test/integration/compute/test_servers.rb → ...tion/compute/core_compute/test_servers.rb b/test/integration/compute/test_servers.rb → ...tion/compute/core_compute/test_servers.rb
@@ -33,9 +33,10 @@ def test_bootstrap
     user = "username"
 
     File.stub :read, key do
-      # Name is set this way so it will be cleaned up by CollectionFactory
+      # XXX Small hack - name is set this way so it will be cleaned up by CollectionFactory
+      # Bootstrap is special so this is something that needs to be done only for this method
       # Public_key_path is set to avoid stubbing out File.exist?
-      server = @subject.bootstrap(:name => "#{CollectionFactory::PREFIX}-#{Time.now.to_i}",
+      server = @subject.bootstrap(:name => "#{CollectionFactory.new(nil,namespaced_name).resource_name}",
                                   :username => user,
                                   :public_key_path => "foo")
       boot_disk = server.disks.detect { |disk| disk[:boot] }
@@ -56,7 +57,9 @@ def test_bootstrap_fail
     # Pretend the ssh key does not exist
     File.stub :exist?, nil do
       assert_raises(Fog::Errors::Error) {
-        @subject.bootstrap(:name => "#{CollectionFactory::PREFIX}-#{Time.now.to_i}",
+        # XXX Small hack - name is set this way so it will be cleaned up by CollectionFactory
+        # Bootstrap is special so this is something that needs to be done only for this method
+        @subject.bootstrap(:name => "#{CollectionFactory.new(nil,namespaced_name).resource_name}",
                            :public_key_path => nil)
       }
     end

diff --git a/test/integration/compute/test_snapshots.rb → ...on/compute/core_compute/test_snapshots.rb b/test/integration/compute/test_snapshots.rb → ...on/compute/core_compute/test_snapshots.rb
diff --git a/test/integration/compute/test_zones.rb → ...ration/compute/core_compute/test_zones.rb b/test/integration/compute/test_zones.rb → ...ration/compute/core_compute/test_zones.rb
diff --git a/test/integration/compute/test_addresses.rb → ...compute/core_networking/test_addresses.rb b/test/integration/compute/test_addresses.rb → ...compute/core_networking/test_addresses.rb
diff --git a/test/integration/compute/core_networking/test_coverage.rb b/test/integration/compute/core_networking/test_coverage.rb
@@ -0,0 +1,6 @@
+require "helpers/integration_test_helper"
+
+# This is a simple coverage helper that helps differentiate
+# the tests when run in parallel so the final coverage report
+# can be properly combined together from multiple runners
+SimpleCov.command_name "test:compute-core_networking" if ENV["COVERAGE"]
diff --git a/test/integration/compute/test_firewalls.rb → ...compute/core_networking/test_firewalls.rb b/test/integration/compute/test_firewalls.rb → ...compute/core_networking/test_firewalls.rb
diff --git a/test/integration/compute/test_networks.rb → .../compute/core_networking/test_networks.rb b/test/integration/compute/test_networks.rb → .../compute/core_networking/test_networks.rb
diff --git a/test/integration/compute/test_routes.rb → ...on/compute/core_networking/test_routes.rb b/test/integration/compute/test_routes.rb → ...on/compute/core_networking/test_routes.rb
diff --git a/test/integration/compute/test_subnetworks.rb → ...mpute/core_networking/test_subnetworks.rb b/test/integration/compute/test_subnetworks.rb → ...mpute/core_networking/test_subnetworks.rb
diff --git a/test/integration/compute/instance_groups/test_coverage.rb b/test/integration/compute/instance_groups/test_coverage.rb
@@ -0,0 +1,6 @@
+require "helpers/integration_test_helper"
+
+# This is a simple coverage helper that helps differentiate
+# the tests when run in parallel so the final coverage report
+# can be properly combined together from multiple runners
+SimpleCov.command_name "test:compute-instance_groups" if ENV["COVERAGE"]
diff --git a/...n/compute/test_instance_group_managers.rb → ...ce_groups/test_instance_group_managers.rb b/...n/compute/test_instance_group_managers.rb → ...ce_groups/test_instance_group_managers.rb
diff --git a/...tegration/compute/test_instance_groups.rb → ...e/instance_groups/test_instance_groups.rb b/...tegration/compute/test_instance_groups.rb → ...e/instance_groups/test_instance_groups.rb
diff --git a/...ration/compute/test_instance_templates.rb → ...nstance_groups/test_instance_templates.rb b/...ration/compute/test_instance_templates.rb → ...nstance_groups/test_instance_templates.rb
diff --git a/...egration/compute/test_backend_services.rb → ...te/loadbalancing/test_backend_services.rb b/...egration/compute/test_backend_services.rb → ...te/loadbalancing/test_backend_services.rb
diff --git a/test/integration/compute/loadbalancing/test_coverage.rb b/test/integration/compute/loadbalancing/test_coverage.rb
@@ -0,0 +1,6 @@
+require "helpers/integration_test_helper"
+
+# This is a simple coverage helper that helps differentiate
+# the tests when run in parallel so the final coverage report
+# can be properly combined together from multiple runners
+SimpleCov.command_name "test:compute-loadbalancing" if ENV["COVERAGE"]
diff --git a/...egration/compute/test_forwarding_rules.rb → ...te/loadbalancing/test_forwarding_rules.rb b/...egration/compute/test_forwarding_rules.rb → ...te/loadbalancing/test_forwarding_rules.rb
diff --git a/...egration/compute/test_global_addresses.rb → ...te/loadbalancing/test_global_addresses.rb b/...egration/compute/test_global_addresses.rb → ...te/loadbalancing/test_global_addresses.rb
diff --git a/...n/compute/test_global_forwarding_rules.rb → ...balancing/test_global_forwarding_rules.rb b/...n/compute/test_global_forwarding_rules.rb → ...balancing/test_global_forwarding_rules.rb
diff --git a/...ration/compute/test_http_health_checks.rb → .../loadbalancing/test_http_health_checks.rb b/...ration/compute/test_http_health_checks.rb → .../loadbalancing/test_http_health_checks.rb
diff --git a/...egration/compute/test_ssl_certificates.rb → ...te/loadbalancing/test_ssl_certificates.rb b/...egration/compute/test_ssl_certificates.rb → ...te/loadbalancing/test_ssl_certificates.rb
diff --git a/...ation/compute/test_target_http_proxies.rb → ...loadbalancing/test_target_http_proxies.rb b/...ation/compute/test_target_http_proxies.rb → ...loadbalancing/test_target_http_proxies.rb
diff --git a/...tion/compute/test_target_https_proxies.rb → ...oadbalancing/test_target_https_proxies.rb b/...tion/compute/test_target_https_proxies.rb → ...oadbalancing/test_target_https_proxies.rb
diff --git a/...egration/compute/test_target_instances.rb → ...te/loadbalancing/test_target_instances.rb b/...egration/compute/test_target_instances.rb → ...te/loadbalancing/test_target_instances.rb