diff --git a/nginx_stage/lib/nginx_stage.rb b/nginx_stage/lib/nginx_stage.rb index 367be473ce..48ea4509d6 100644 --- a/nginx_stage/lib/nginx_stage.rb +++ b/nginx_stage/lib/nginx_stage.rb @@ -121,6 +121,13 @@ def self.parse_app_request(request:) app_info end + # Expand the mTLS PKI directory for a given user + # @param user [User, String] the user + # @return [String] absolute path to the user's mTLS PKI directory + def self.mtls_pki_dir(user:) + pun_mtls_pki_dir.sub('~', Etc.getpwnam(user.to_s).dir) + end + # Clean environment used during execution of nginx binary # @example Start the per-user NGINX for user Bob # clean_nginx_env(user: 'bob') @@ -128,6 +135,8 @@ def self.parse_app_request(request:) # @param user [String] the owner of the nginx process # @return [ENV] the environment used to execute the nginx process def self.clean_nginx_env(user:) + pki_dir = mtls_pki_dir(user: user) + ENV.replace({ "USER" => user, "LOGNAME" => user, @@ -148,6 +157,9 @@ def self.clean_nginx_env(user:) "ALLOWED_HOSTS" => ENV['OOD_ALLOWED_HOSTS'], # set the duplicate to keep clean_nginx_env idempotent "OOD_ALLOWED_HOSTS" => ENV['OOD_ALLOWED_HOSTS'], + "ONDEMAND_MTLS_CERT" => File.join(pki_dir, 'client.crt'), + "ONDEMAND_MTLS_KEY" => File.join(pki_dir, 'client.key'), + "ONDEMAND_MTLS_CA" => File.join(pki_dir, 'ca.crt'), }.merge(pun_custom_env).merge(preserve_env_declarations.map { |k| [ k, ENV[k] ] }.to_h)) end diff --git a/nginx_stage/lib/nginx_stage/configuration.rb b/nginx_stage/lib/nginx_stage/configuration.rb index b1233bc7e5..b232ee3271 100644 --- a/nginx_stage/lib/nginx_stage/configuration.rb +++ b/nginx_stage/lib/nginx_stage/configuration.rb @@ -412,6 +412,27 @@ def user_regex # @return [String] path to the custom html root attr_accessor :pun_custom_html_root + # Whether mTLS proxying from the PUN to compute nodes is enabled + # @return [Boolean] mTLS proxy enabled + attr_accessor :pun_mtls_enabled + + # Directory under the user's home for storing mTLS PKI material + # @return [String] path pattern for mTLS PKI directory + attr_accessor :pun_mtls_pki_dir + + # Validity period in days for the generated mTLS certificate + # @return [Integer] certificate validity in days + attr_accessor :pun_mtls_cert_days + + # Regex used to validate target hosts for mTLS proxying (SSRF protection) + # @return [String] host regex pattern + attr_accessor :pun_mtls_host_regex + + # DNS resolver address(es) for nginx to use with variable-based proxy_pass. + # Auto-detected from /etc/resolv.conf if not set. + # @return [String] resolver address(es) + attr_accessor :pun_mtls_resolver + # # Configuration module # @@ -528,6 +549,19 @@ def set_default_configuration self.disable_bundle_user_config = true self.nginx_file_upload_max = '10737420000' + self.pun_mtls_enabled = false + self.pun_mtls_pki_dir = '~/ondemand/pki' + self.pun_mtls_cert_days = 3650 + self.pun_mtls_host_regex = '[\w.-]+' + self.pun_mtls_resolver = begin + nameserver = File.readlines('/etc/resolv.conf') + .grep(/^\s*nameserver\s+/) + .first&.split&.last + nameserver || '127.0.0.1' + rescue + '127.0.0.1' + end + read_configuration(default_config_path) if File.file?(default_config_path) end diff --git a/nginx_stage/lib/nginx_stage/generators/pun_config_generator.rb b/nginx_stage/lib/nginx_stage/generators/pun_config_generator.rb index 390b0fb908..436e2ae408 100644 --- a/nginx_stage/lib/nginx_stage/generators/pun_config_generator.rb +++ b/nginx_stage/lib/nginx_stage/generators/pun_config_generator.rb @@ -105,6 +105,79 @@ class PunConfigGenerator < Generator end end + # Generate per-user mTLS keypair if enabled and not already present or expired. + # Runs file operations in a forked child that drops privileges to the target + # user so that root_squash NFS shares are handled correctly. + add_hook :create_mtls_keypair do + if NginxStage.pun_mtls_enabled + require 'openssl' + require 'fileutils' + + pki_dir = NginxStage.mtls_pki_dir(user: user) + cert_path = File.join(pki_dir, 'client.crt') + key_path = File.join(pki_dir, 'client.key') + ca_path = File.join(pki_dir, 'ca.crt') + + # Check if cert exists and is not expired + needs_generation = true + if File.file?(cert_path) && File.file?(key_path) + begin + existing_cert = OpenSSL::X509::Certificate.new(File.read(cert_path)) + needs_generation = false if existing_cert.not_after > Time.now + rescue OpenSSL::X509::CertificateError + # Corrupt cert, regenerate + end + end + + if needs_generation + # Generate key material in the parent (no filesystem access needed) + key = OpenSSL::PKey::EC.generate('prime256v1') + + cert = OpenSSL::X509::Certificate.new + cert.version = 2 + cert.serial = OpenSSL::BN.rand(128) + cert.subject = OpenSSL::X509::Name.new([['CN', "#{user}@ondemand"]]) + cert.issuer = cert.subject + cert.public_key = key + cert.not_before = Time.now - 60 + cert.not_after = Time.now + (NginxStage.pun_mtls_cert_days * 86400) + + ef = OpenSSL::X509::ExtensionFactory.new + ef.subject_certificate = cert + ef.issuer_certificate = cert + cert.add_extension(ef.create_extension('basicConstraints', 'CA:TRUE', true)) + cert.add_extension(ef.create_extension('subjectKeyIdentifier', 'hash')) + cert.add_extension(ef.create_extension('keyUsage', 'digitalSignature,keyCertSign', true)) + + cert.sign(key, OpenSSL::Digest::SHA256.new) + + key_pem = key.to_pem + cert_pem = cert.to_pem + + # Fork a child that drops to the user's UID/GID before writing files, + # so NFS root_squash shares see the real user, not root/nobody. + pw = Etc.getpwnam(user.to_s) + pid = Process.fork do + Process::GID.change_privilege(pw.gid) + Process::UID.change_privilege(pw.uid) + + FileUtils.mkdir_p(pki_dir, mode: 0700) + + File.write(key_path, key_pem) + File.chmod(0600, key_path) + + File.write(cert_path, cert_pem) + File.chmod(0644, cert_path) + + FileUtils.cp(cert_path, ca_path) + File.chmod(0644, ca_path) + end + _, status = Process.waitpid2(pid) + $stderr.puts "WARNING: mTLS keypair generation failed for #{user}" unless status.success? + end + end + end + # Run the pre hook command. This eats the output and doesn't affect # the overall status of the PUN startup # This must come before anything that cleans the process environment diff --git a/nginx_stage/lib/nginx_stage/views/pun_config_view.rb b/nginx_stage/lib/nginx_stage/views/pun_config_view.rb index f9d57a7746..ca98973128 100644 --- a/nginx_stage/lib/nginx_stage/views/pun_config_view.rb +++ b/nginx_stage/lib/nginx_stage/views/pun_config_view.rb @@ -145,6 +145,70 @@ def disable_bundle_user_config? NginxStage.disable_bundle_user_config end + # Whether mTLS proxying to compute nodes is enabled + # @return [Boolean] mTLS proxy enabled + def mtls_enabled? + NginxStage.pun_mtls_enabled + end + + # Path to the mTLS client certificate + # @return [String] path to client cert + def mtls_client_cert_path + File.join(NginxStage.mtls_pki_dir(user: user), 'client.crt') + end + + # Path to the mTLS client private key + # @return [String] path to client key + def mtls_client_key_path + File.join(NginxStage.mtls_pki_dir(user: user), 'client.key') + end + + # Path to the mTLS CA certificate + # @return [String] path to CA cert + def mtls_ca_cert_path + File.join(NginxStage.mtls_pki_dir(user: user), 'ca.crt') + end + + # Regex pattern for allowed mTLS proxy target hosts + # @return [String] host regex + def mtls_host_regex + NginxStage.pun_mtls_host_regex + end + + # DNS resolver for nginx variable-based proxy_pass + # @return [String] resolver address(es) + def mtls_resolver + NginxStage.pun_mtls_resolver + end + + # Expected CN of the upstream server's certificate for SSL name verification + # Headers to strip when proxying to compute nodes + # @return [Array] list of header names + def strip_proxy_headers + %w[ + Authorization + X-Forwarded-User + OIDC_CLAIM_sub + OIDC_CLAIM_preferred_username + OIDC_CLAIM_given_name + OIDC_CLAIM_zoneinfo + OIDC_CLAIM_locale + OIDC_CLAIM_email + OIDC_CLAIM_email_verified + OIDC_CLAIM_iss + OIDC_CLAIM_nonce + OIDC_CLAIM_aud + OIDC_CLAIM_acr + OIDC_CLAIM_azp + OIDC_CLAIM_auth_time + OIDC_CLAIM_exp + OIDC_CLAIM_iat + OIDC_CLAIM_jti + OIDC_access_token + OIDC_access_token_expires + ] + end + # View used to confirm whether the user wants to restart the PUN to reload # configuration changes # @return [String] restart confirmation view diff --git a/nginx_stage/share/nginx_stage_example.yml b/nginx_stage/share/nginx_stage_example.yml index 4ddd627508..f4ad3663ef 100644 --- a/nginx_stage/share/nginx_stage_example.yml +++ b/nginx_stage/share/nginx_stage_example.yml @@ -52,6 +52,22 @@ # error page can be customized with this mechanism. #pun_custom_html_root: '/etc/ood/config/pun/html' +# Whether mTLS proxying from the PUN to compute nodes is enabled +#pun_mtls_enabled: false + +# Directory under the user's home for storing mTLS PKI material +#pun_mtls_pki_dir: '~/ondemand/pki' + +# Validity period in days for the generated mTLS certificate +#pun_mtls_cert_days: 3650 + +# Regex used to validate target hosts for mTLS proxying (SSRF protection) +#pun_mtls_host_regex: '[\w.-]+' + +# DNS resolver for nginx to use with variable-based proxy_pass +# Auto-detected from /etc/resolv.conf if not set +#pun_mtls_resolver: '8.8.8.8' + # Location of the ERB templates used in the generation of the NGINX configs # #template_root: '/opt/ood/nginx_stage/templates' diff --git a/nginx_stage/templates/pun.conf.erb b/nginx_stage/templates/pun.conf.erb index 86a88bee65..74d42ee8ef 100644 --- a/nginx_stage/templates/pun.conf.erb +++ b/nginx_stage/templates/pun.conf.erb @@ -78,6 +78,14 @@ http { default_type application/octet-stream; + <%- if mtls_enabled? -%> + # WebSocket upgrade map for mTLS proxy + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + <%- end -%> + log_format main '<%= log_format %>'; access_log <%= access_log_path %> main; @@ -132,5 +140,96 @@ http { <%- app_configs.each do |app_config| -%> include <%= app_config %>; <%- end -%> + + <%- if mtls_enabled? -%> + # mTLS proxy to compute node services (full path preserved) + location ~ ^/pun/proxy/(?<%= mtls_host_regex %>)/(?\d+)(?/.*)?$ { + + # Resolver needed for variable-based proxy_pass + resolver <%= mtls_resolver %> valid=300s; + resolver_timeout 5s; + + # Strip sensitive headers before proxying + <%- strip_proxy_headers.each do |header| -%> + proxy_set_header <%= header %> ""; + <%- end -%> + + # Forward useful context headers + proxy_set_header Host $target_host:$target_port; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + + # WebSocket support + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_http_version 1.1; + + # mTLS client certificate configuration + proxy_ssl_certificate <%= mtls_client_cert_path %>; + proxy_ssl_certificate_key <%= mtls_client_key_path %>; + proxy_ssl_trusted_certificate <%= mtls_ca_cert_path %>; + proxy_ssl_verify on; + proxy_ssl_verify_depth 1; + proxy_ssl_server_name on; + proxy_ssl_name $target_host; + proxy_ssl_protocols TLSv1.2 TLSv1.3; + + # URL rewriting for Location headers + proxy_redirect https://$target_host:$target_port/ /pun/proxy/$target_host/$target_port/; + + # Rewrite Set-Cookie domain and path + proxy_cookie_domain $target_host localhost; + proxy_cookie_path / /pun/proxy/$target_host/$target_port/; + + # Preserve the full request path including /pun/proxy// + proxy_pass https://$target_host:$target_port$target_url$request_uri; + } + + # mTLS relative proxy to compute node services (path prefix stripped) + location ~ ^/pun/rproxy/(?<%= mtls_host_regex %>)/(?\d+)(?/.*)?$ { + + # Resolver needed for variable-based proxy_pass + resolver <%= mtls_resolver %> valid=300s; + resolver_timeout 5s; + + # Strip sensitive headers before proxying + <%- strip_proxy_headers.each do |header| -%> + proxy_set_header <%= header %> ""; + <%- end -%> + + # Forward useful context headers + proxy_set_header Host $target_host:$target_port; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Prefix /pun/rproxy/$target_host/$target_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + + # WebSocket support + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_http_version 1.1; + + # mTLS client certificate configuration + proxy_ssl_certificate <%= mtls_client_cert_path %>; + proxy_ssl_certificate_key <%= mtls_client_key_path %>; + proxy_ssl_trusted_certificate <%= mtls_ca_cert_path %>; + proxy_ssl_verify on; + proxy_ssl_verify_depth 1; + proxy_ssl_server_name on; + proxy_ssl_name $target_host; + proxy_ssl_protocols TLSv1.2 TLSv1.3; + + # URL rewriting for Location headers + proxy_redirect https://$target_host:$target_port/ /pun/rproxy/$target_host/$target_port/; + + # Rewrite Set-Cookie domain and path + proxy_cookie_domain $target_host localhost; + proxy_cookie_path / /pun/rproxy/$target_host/$target_port/; + + # Proxy to compute node over TLS + proxy_pass https://$target_host:$target_port$target_url$target_uri$is_args$args; + } + <%- end -%> } } diff --git a/ood-portal-generator/lib/ood_portal_generator/view.rb b/ood-portal-generator/lib/ood_portal_generator/view.rb index 74c3daf2f8..ce5f9d95c9 100644 --- a/ood-portal-generator/lib/ood_portal_generator/view.rb +++ b/ood-portal-generator/lib/ood_portal_generator/view.rb @@ -96,6 +96,10 @@ def initialize(opts = {}) @pun_pre_hook_exports = opts.fetch(:pun_pre_hook_exports, nil) @pun_pre_hook_root_cmd = opts.fetch(:pun_pre_hook_root_cmd, nil) + # mTLS proxy sub-uri (header stripping at Apache layer) + @pun_proxy_uri = opts.fetch(:pun_proxy_uri, nil) + @pun_rproxy_uri = opts.fetch(:pun_rproxy_uri, @pun_proxy_uri ? "/pun/rproxy" : nil) + # OpenID Connect sub-uri @oidc_uri = opts.fetch(:oidc_uri, nil) @oidc_discover_uri = opts.fetch(:oidc_discover_uri, nil) diff --git a/ood-portal-generator/share/ood_portal_example.yml b/ood-portal-generator/share/ood_portal_example.yml index 4c72583534..4dd9c621f8 100644 --- a/ood-portal-generator/share/ood_portal_example.yml +++ b/ood-portal-generator/share/ood_portal_example.yml @@ -337,6 +337,21 @@ # Default: null (pass nothing) #pun_pre_hook_exports: null +# Sub-uri for mTLS proxy requests. When set, Apache strips sensitive headers +# (Authorization, OIDC claims, session cookies) from requests to this path +# before they reach the PUN. +# Example: +# pun_proxy_uri: '/pun/proxy' +# Default: null (disabled) +#pun_proxy_uri: null + +# Sub-uri for mTLS relative proxy requests. This behaves like rnode/rproxy by +# stripping the /pun/rproxy// prefix before forwarding upstream. +# Example: +# pun_rproxy_uri: '/pun/rproxy' +# Default: '/pun/rproxy' when pun_proxy_uri is set, otherwise null +#pun_rproxy_uri: null + # # Support for OpenID Connect # diff --git a/ood-portal-generator/templates/ood-portal.conf.erb b/ood-portal-generator/templates/ood-portal.conf.erb index e6380f3c1a..a00e76484d 100644 --- a/ood-portal-generator/templates/ood-portal.conf.erb +++ b/ood-portal-generator/templates/ood-portal.conf.erb @@ -406,6 +406,20 @@ Listen <%= addr_port %> LuaHookFixups pun_proxy.lua pun_proxy_handler + <%- if @pun_proxy_uri || @pun_rproxy_uri -%> + + # Strip sensitive headers from mTLS proxy requests before they reach the PUN + <%- [@pun_proxy_uri, @pun_rproxy_uri].compact.each do |proxy_uri| -%> + "> + <%- @strip_proxy_cookies.to_a.each do |cookie| -%> + RequestHeader edit* Cookie "<%= cookie %>=[^;]+;" "" + <%- end -%> + <%- @strip_proxy_headers.to_a.each do |header| -%> + RequestHeader unset <%= header %> + <%- end -%> + + <%- end -%> + <%- end -%> # Control backend PUN for authenticated user: # NB: See mod_ood_proxy for more details.