Merge pull request #3548 from OSC/update-3.1

Update 3.1

.github/workflows/tests.yml

@@ -99,8 +99,7 @@ jobs:
       - name: Get ondemand token
         id: token
         run: |
-           TOKEN_NAME=$(kubectl describe serviceaccount ondemand -n ondemand | grep Tokens | awk '{ print $2 }')
-           TOKEN=$(kubectl describe secret $TOKEN_NAME -n ondemand | grep "token:" | awk '{ print $2 }')
+           TOKEN=$(kubectl create token ondemand --namespace=ondemand)
            echo "ondemand=${TOKEN}" >> $GITHUB_OUTPUT
       - name: Setup kubectl
         run: |

apps/dashboard/app/javascript/dynamic_forms.js

@@ -100,7 +100,8 @@ function snakeCaseWords(str) {
       snakeCase += c.toLowerCase();
     } else if(c == c.toUpperCase() && isNaN(c)) {
       const nextIsUpper = (index + 1 !== str.length) ? str[index + 1] === str[index + 1].toUpperCase() : true;
-      if (str[index-1] === '_' || nextIsUpper) {
+      const nextIsNum = !isNaN(str[index + 1]);
+      if ((str[index-1] === '_' || nextIsUpper) && !nextIsNum) {
         snakeCase += c.toLowerCase();
       } else {
         snakeCase += `_${c.toLowerCase()}`;

apps/dashboard/test/fixtures/sys_with_gateway_apps/bc_jupyter/form.yml

@@ -38,6 +38,7 @@ attributes:
           data-max-bc-num-slots-for-cluster-oakley: 8,
           data-min-gpus: 0,
           data-max-gpus: 0,
+          data-hide-gpus-num-v100: true,
         ]
       - [
           "broken",
@@ -46,6 +47,7 @@ attributes:
           data-maximum-bc-not-found-for-cluster-mistype: 30,
           data-min-gpus: 0,
           data-max-gpus: 0,
+          data-hide-gpus-num-v100: true,
         ]
       - [
           "gpu",
@@ -76,6 +78,7 @@ attributes:
 
           data-min-gpus: 0,
           data-max-gpus: 0,
+          data-hide-gpus-num-v100: true,
         ]
       - [
           "advanced",
@@ -85,6 +88,7 @@ attributes:
 
           data-min-gpus: 0,
           data-max-gpus: 0,
+          data-hide-gpus-num-v100: true,
         ]
       # this node type is the same for both clusters, so there's no 'for-cluster-...' clause
       - [
@@ -99,6 +103,7 @@ attributes:
 
           data-min-gpus: 0,
           data-max-gpus: 0,
+          data-hide-gpus-num-v100: true,
         ]
       - [
           "other-40ish-option",
@@ -108,6 +113,7 @@ attributes:
 
           data-min-gpus: 0,
           data-max-gpus: 0,
+          data-hide-gpus-num-v100: true,
         ]
   python_version:
     # let's set the account used by the python version for some reason
@@ -214,3 +220,4 @@ form:
   - auto_modules_intel
   - auto_modules_netcdf-serial
   - checkbox_test
+  - gpus_num_v100

apps/dashboard/test/models/batch_connect/session_test.rb

@@ -599,7 +599,8 @@ def completed?
           'auto_modules_app_jupyter'   => '',
           'auto_modules_intel'         => '',
           'auto_modules_netcdf_serial' => '',
-          'checkbox_test'              => ''
+          'checkbox_test'              => '',
+          'gpus_num_v100'              => ''
         }
 
         assert session.save(app: bc_jupyter_app, context: ctx), session.errors.each(&:to_s).to_s

apps/dashboard/test/system/batch_connect_test.rb

@@ -674,6 +674,18 @@ def make_bc_app(dir, form)
     assert_equal 'display: none;', find_option_style('classroom_size', 'large')
   end
 
+  test 'can hide fields with numbers and characters' do
+    visit new_batch_connect_session_context_url('sys/bc_jupyter')
+
+    # defaults - gpus_num_v100 is hidden on page load.
+    assert_equal('any', find_value('node_type'))
+    refute(find("##{bc_ele_id('gpus_num_v100')}", visible: false).visible?)
+
+    # select gpu and now it's shown.
+    select('gpu', from: bc_ele_id('node_type'))
+    assert(find("##{bc_ele_id('gpus_num_v100')}").visible?)
+  end
+
   test 'options can check and uncheck' do
     visit new_batch_connect_session_context_url('sys/bc_jupyter')
 

nginx_stage/lib/nginx_stage.rb

@@ -5,6 +5,7 @@
 require_relative "nginx_stage/pid_file"
 require_relative "nginx_stage/socket_file"
 require_relative "nginx_stage/secret_key_base_file"
+require_relative "nginx_stage/session_finder"
 require_relative "nginx_stage/views/pun_config_view"
 require_relative "nginx_stage/views/app_config_view"
 require_relative "nginx_stage/generator"

nginx_stage/lib/nginx_stage/generators/nginx_clean_generator.rb

@@ -2,6 +2,9 @@ module NginxStage
   # This generator cleans all running per-user NGINX processes that are
   # inactive (i.e., not active connections).
   class NginxCleanGenerator < Generator
+
+    include NginxStage::SessionFinder
+
     desc 'Clean all user running PUNs with no active connections'
 
     footer <<-EOF.gsub(/^ {4}/, '')
@@ -59,8 +62,9 @@ class NginxCleanGenerator < Generator
           next if (user && user != u.to_s)
           pid_path = PidFile.new NginxStage.pun_pid_path(user: u)
           socket = SocketFile.new NginxStage.pun_socket_path(user: u)
-          cleanup_stale_files(pid_path, socket) unless pid_path.running_process? 
-          if socket.sessions.zero? || force
+          sessions = session_count(u)
+          cleanup_stale_files(pid_path, socket) unless pid_path.running_process?
+          if sessions.zero? || force
             puts u
             if !skip_nginx
               NginxStage.clean_nginx_env(user: user)

nginx_stage/lib/nginx_stage/generators/nginx_show_generator.rb

@@ -1,6 +1,9 @@
 module NginxStage
   # This generator shows the state of the running per-user NGINX process.
   class NginxShowGenerator < Generator
+
+    include NginxStage::SessionFinder
+
     desc 'Show the details for a given per-user nginx process'
 
     footer <<-EOF.gsub(/^ {4}/, '')
@@ -31,7 +34,7 @@ class NginxShowGenerator < Generator
     add_hook :check_socket_for_active_sessions do
       socket = SocketFile.new socket_path
       puts "Socket: #{socket}"
-      puts "Sessions: #{socket.sessions}"
+      puts "Sessions: #{session_count(user)}"
     end
 
     private

nginx_stage/lib/nginx_stage/session_finder.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module NginxStage
+  module SessionFinder
+    def session_count(user)
+      `timeout 10 ps -o cmd -u #{user}`.split("\n").select do |command|
+        # matches 'Passenger NodeApp', 'Passenger RubyApp' and so on.
+        command.match?(/Passenger \w+App:/)
+      end.count
+    end
+  end
+end

nginx_stage/lib/nginx_stage/socket_file.rb

@@ -10,18 +10,6 @@ def initialize(socket)
       @socket = Pathname.new(socket)
       raise MissingSocketFile, "missing socket file: #{socket}" unless File.exist?(socket)
       raise InvalidSocketFile, "invalid socket file: #{socket}" unless File.socket?(socket)
-      @processes = get_processes
-    end
-
-    # The number of active sessions connected to this socket
-    # @return [Integer] number of active connections
-    def sessions
-      # generate array of inodes
-      ary_inodes = @processes.map{|h| h[:inode]}.reduce([], :+)
-
-      # count number of inodes without partner (assuming these are connected to
-      # apache proxy instead of root nginx process)
-      ary_inodes.group_by{|e| e}.select{|k,v| v.size == 1}.map(&:first).count
     end
 
     # Convert object to string
@@ -43,20 +31,5 @@ def delete
       $stderr.puts "Unable to delete socket file at #{socket}"
     end
 
-    private
-      def get_processes
-        str = `lsof -F piu #{socket}`
-        ary = []
-        str.split(/\n/).each do |l|
-          if /^p(?<pid>\d+)$/ =~ l
-            ary << {pid: pid, uid: nil, inode: []}
-          elsif /^u(?<uid>\d+)$/ =~ l
-            ary.last[:uid] = uid
-          elsif /^i(?<inode>\d+)$/ =~ l
-            ary.last[:inode] << inode
-          end
-        end
-        ary
-      end
   end
 end

ood-portal-generator/lib/ood_portal_generator/dex.rb

@@ -297,7 +297,6 @@ def oidc_attributes
       attrs = {
         dex_http_port:              http_port,
         oidc_uri:                   '/oidc',
-        oidc_redirect_uri:          client_redirect_uri,
         oidc_provider_metadata_url: "#{issuer}/.well-known/openid-configuration",
         oidc_client_id:             client_id,
         oidc_client_secret:         client_secret

ood-portal-generator/lib/ood_portal_generator/view.rb

@@ -7,7 +7,7 @@ module OodPortalGenerator
   class View
     attr_reader :ssl, :protocol, :proxy_server, :port, :dex_uri
     attr_accessor :user_map_match, :user_map_cmd, :logout_redirect, :dex_http_port, :dex_enabled
-    attr_accessor :oidc_uri, :oidc_client_secret, :oidc_remote_user_claim, :oidc_client_id, :oidc_provider_metadata_url, :oidc_redirect_uri
+    attr_accessor :oidc_uri, :oidc_client_secret, :oidc_remote_user_claim, :oidc_client_id, :oidc_provider_metadata_url
 
     # let the application set the auth if it needs to
     attr_writer :auth
@@ -39,6 +39,7 @@ def initialize(opts = {})
       @errorlog         = log_filename(error_log,"error")
       @logformat        = opts.fetch(:logformat, nil)
       @use_rewrites     = opts.fetch(:use_rewrites, true)
+      @http_redirect_host = opts.fetch(:http_redirect_host, '%{HTTP_HOST}')
       @lua_root         = opts.fetch(:lua_root, "/opt/ood/mod_ood_proxy/lib")
       @lua_log_level    = opts.fetch(:lua_log_level, "info")
       @user_map_cmd     = opts.fetch(:user_map_cmd, nil)
@@ -107,7 +108,6 @@ def initialize(opts = {})
       @oidc_provider_metadata_url       = opts.fetch(:oidc_provider_metadata_url, nil)
       @oidc_client_id                   = opts.fetch(:oidc_client_id, nil)
       @oidc_client_secret               = opts.fetch(:oidc_client_secret, nil)
-      @oidc_redirect_uri                = "#{protocol}#{servername}#{@oidc_uri}"
       @oidc_remote_user_claim           = opts.fetch(:oidc_remote_user_claim, 'preferred_username')
       @oidc_scope                       = opts.fetch(:oidc_scope, "openid profile email")
       @oidc_crypto_passphrase           = opts.fetch(:oidc_crypto_passphrase, Digest::SHA1.hexdigest(servername))

ood-portal-generator/share/ood_portal_example.yml

@@ -80,6 +80,12 @@
 # Default: true
 #use_rewrites: true
 
+# Specify the host to redirect to when redirecting from port 80
+# Example:
+#     http_redirect_host: my.proxy.host
+# Default: '%{HTTP_HOST}'
+#http_redirect_host: '%{HTTP_HOST}'
+
 # Should Maintenance Rewrite rules be added
 # Example:
 #   use_maintenance: false

ood-portal-generator/spec/application_spec.rb

@@ -132,6 +132,10 @@ def test_generate(input, output)
       test_generate('input/custom_directives.yml', 'output/custom_directives.conf')
     end
 
+    it 'http_redirect_host can be set' do
+      test_generate('input/http_redirect_host.yml', 'output/http_redirect_host.conf')
+    end
+
     it 'generates full OIDC config' do
       config = {
         servername: 'ondemand.example.com',

ood-portal-generator/spec/fixtures/input/http_redirect_host.yml

@@ -0,0 +1,14 @@
+---
+auth:
+ - 'AuthType openid-connect'
+ - 'Require valid-user'
+
+servername: ondemand.example.com
+proxy_server: ondemand.proxy.example.com
+http_redirect_host: ondemand.redirect-proxy.example.com
+
+port: 443
+ssl:
+  - 'SSLCertificateFile /etc/pki/tls/certs/ondemand.example.com.crt'
+  - 'SSLCertificateKeyFile /etc/pki/tls/private/ondemand.example.com.key'
+  - 'SSLCertificateChainFile /etc/pki/tls/certs/ondemand.example.com-interm.crt'

ood-portal-generator/spec/fixtures/ood-portal.conf.all

@@ -54,7 +54,7 @@ Listen 8080
   ServerAlias foo.example.com
 
   RewriteEngine On
-  RewriteRule ^(.*) https://test.proxy.name:8080$1 [R=301,NE,L]
+  RewriteRule ^(.*) https://%{HTTP_HOST}:8080$1 [R=301,NE,L]
 </VirtualHost>
 
 # The Open OnDemand portal VirtualHost

ood-portal-generator/spec/fixtures/ood-portal.conf.dex

@@ -64,7 +64,7 @@
   OIDCProviderMetadataURL http://example.com/dex/.well-known/openid-configuration
   OIDCClientID example.com
   OIDCClientSecret 83bc78b7-6f5e-4010-9d80-22f328aa6550
-  OIDCRedirectURI http://example.com/oidc
+  OIDCRedirectURI /oidc
   OIDCRemoteUserClaim email
   OIDCScope "openid profile email"
   OIDCCryptoPassphrase 0caaf24ab1a0c33440c06afe99df986365b0781f

ood-portal-generator/spec/fixtures/ood-portal.conf.dex-full

@@ -47,7 +47,7 @@
   ServerName example.com
 
   RewriteEngine On
-  RewriteRule ^(.*) https://example.com:443$1 [R=301,NE,L]
+  RewriteRule ^(.*) https://%{HTTP_HOST}:443$1 [R=301,NE,L]
 </VirtualHost>
 
 # The Open OnDemand portal VirtualHost
@@ -84,7 +84,7 @@
   OIDCProviderMetadataURL https://example.com/dex/.well-known/openid-configuration
   OIDCClientID example.com
   OIDCClientSecret 83bc78b7-6f5e-4010-9d80-22f328aa6550
-  OIDCRedirectURI https://example.com/oidc
+  OIDCRedirectURI /oidc
   OIDCRemoteUserClaim email
   OIDCScope "openid profile email"
   OIDCCryptoPassphrase 0caaf24ab1a0c33440c06afe99df986365b0781f

ood-portal-generator/spec/fixtures/ood-portal.conf.dex-ldap

@@ -47,7 +47,7 @@
   ServerName example.com
 
   RewriteEngine On
-  RewriteRule ^(.*) https://example.com:443$1 [R=301,NE,L]
+  RewriteRule ^(.*) https://%{HTTP_HOST}:443$1 [R=301,NE,L]
 </VirtualHost>
 
 # The Open OnDemand portal VirtualHost
@@ -84,7 +84,7 @@
   OIDCProviderMetadataURL https://example.com/dex/.well-known/openid-configuration
   OIDCClientID example.com
   OIDCClientSecret 83bc78b7-6f5e-4010-9d80-22f328aa6550
-  OIDCRedirectURI https://example.com/oidc
+  OIDCRedirectURI /oidc
   OIDCRemoteUserClaim preferred_username
   OIDCScope "openid profile email"
   OIDCCryptoPassphrase 0caaf24ab1a0c33440c06afe99df986365b0781f

ood-portal-generator/spec/fixtures/ood-portal.conf.dex-no-proxy

@@ -47,7 +47,7 @@
   ServerName example.com
 
   RewriteEngine On
-  RewriteRule ^(.*) https://example.com:443$1 [R=301,NE,L]
+  RewriteRule ^(.*) https://%{HTTP_HOST}:443$1 [R=301,NE,L]
 </VirtualHost>
 
 # The Open OnDemand portal VirtualHost
@@ -84,7 +84,7 @@
   OIDCProviderMetadataURL https://example.com:5554/.well-known/openid-configuration
   OIDCClientID example.com
   OIDCClientSecret 83bc78b7-6f5e-4010-9d80-22f328aa6550
-  OIDCRedirectURI https://example.com/oidc
+  OIDCRedirectURI /oidc
   OIDCRemoteUserClaim email
   OIDCScope "openid profile email"
   OIDCCryptoPassphrase 0caaf24ab1a0c33440c06afe99df986365b0781f

ood-portal-generator/spec/fixtures/ood-portal.conf.oidc

@@ -68,7 +68,7 @@
   OIDCProviderMetadataURL https://idp.example.com/auth/realms/osc/.well-known/openid-configuration
   OIDCClientID ondemand.example.com
   OIDCClientSecret secret
-  OIDCRedirectURI http://ondemand.example.com/oidc
+  OIDCRedirectURI /oidc
   OIDCRemoteUserClaim preferred_username
   OIDCScope "openid profile email groups"
   OIDCCryptoPassphrase e2c5ee12c92a019f19b5e532641ac0da2f9acdac

ood-portal-generator/spec/fixtures/ood-portal.conf.oidc-ssl

@@ -47,7 +47,7 @@
   ServerName ondemand.example.com
 
   RewriteEngine On
-  RewriteRule ^(.*) https://ondemand.example.com:443$1 [R=301,NE,L]
+  RewriteRule ^(.*) https://%{HTTP_HOST}:443$1 [R=301,NE,L]
 </VirtualHost>
 
 # The Open OnDemand portal VirtualHost
@@ -84,7 +84,7 @@
   OIDCProviderMetadataURL https://idp.example.com/auth/realms/osc/.well-known/openid-configuration
   OIDCClientID ondemand.example.com
   OIDCClientSecret secret
-  OIDCRedirectURI https://ondemand.example.com/oidc
+  OIDCRedirectURI /oidc
   OIDCRemoteUserClaim preferred_username
   OIDCScope "openid profile email groups"
   OIDCCryptoPassphrase e2c5ee12c92a019f19b5e532641ac0da2f9acdac

ood-portal-generator/spec/fixtures/ood-portal.dex-full.proxy.conf

@@ -47,7 +47,7 @@
   ServerName example.com
 
   RewriteEngine On
-  RewriteRule ^(.*) https://example-proxy.com:443$1 [R=301,NE,L]
+  RewriteRule ^(.*) https://%{HTTP_HOST}:443$1 [R=301,NE,L]
 </VirtualHost>
 
 # The Open OnDemand portal VirtualHost
@@ -84,7 +84,7 @@
   OIDCProviderMetadataURL https://example-proxy.com/dex/.well-known/openid-configuration
   OIDCClientID example.com
   OIDCClientSecret 83bc78b7-6f5e-4010-9d80-22f328aa6550
-  OIDCRedirectURI https://example.com/oidc
+  OIDCRedirectURI /oidc
   OIDCRemoteUserClaim email
   OIDCScope "openid profile email"
   OIDCCryptoPassphrase 0caaf24ab1a0c33440c06afe99df986365b0781f