Add PUN-based mTLS proxy

nginx_stage/lib/nginx_stage.rb

@@ -121,13 +121,22 @@ def self.parse_app_request(request:)
     app_info
   end
 
+  # Expand the mTLS PKI directory for a given user
+  # @param user [User, String] the user
+  # @return [String] absolute path to the user's mTLS PKI directory
+  def self.mtls_pki_dir(user:)
+    pun_mtls_pki_dir.sub('~', Etc.getpwnam(user.to_s).dir)
+  end
+
   # Clean environment used during execution of nginx binary
   # @example Start the per-user NGINX for user Bob
   #   clean_nginx_env(user: 'bob')
   #   #=> { "USER" => "bob", ... }
   # @param user [String] the owner of the nginx process
   # @return [ENV] the environment used to execute the nginx process
   def self.clean_nginx_env(user:)
+    pki_dir = mtls_pki_dir(user: user)
+
     ENV.replace({
       "USER" => user,
       "LOGNAME" => user,
@@ -148,6 +157,9 @@ def self.clean_nginx_env(user:)
       "ALLOWED_HOSTS" => ENV['OOD_ALLOWED_HOSTS'],
       # set the duplicate to keep clean_nginx_env idempotent
       "OOD_ALLOWED_HOSTS" => ENV['OOD_ALLOWED_HOSTS'],
+      "ONDEMAND_MTLS_CERT" => File.join(pki_dir, 'client.crt'),
+      "ONDEMAND_MTLS_KEY"  => File.join(pki_dir, 'client.key'),
+      "ONDEMAND_MTLS_CA"   => File.join(pki_dir, 'ca.crt'),
     }.merge(pun_custom_env).merge(preserve_env_declarations.map { |k| [ k, ENV[k] ] }.to_h))
   end
 

nginx_stage/lib/nginx_stage/configuration.rb

@@ -412,6 +412,27 @@ def user_regex
     # @return [String] path to the custom html root
     attr_accessor :pun_custom_html_root
 
+    # Whether mTLS proxying from the PUN to compute nodes is enabled
+    # @return [Boolean] mTLS proxy enabled
+    attr_accessor :pun_mtls_enabled
+
+    # Directory under the user's home for storing mTLS PKI material
+    # @return [String] path pattern for mTLS PKI directory
+    attr_accessor :pun_mtls_pki_dir
+
+    # Validity period in days for the generated mTLS certificate
+    # @return [Integer] certificate validity in days
+    attr_accessor :pun_mtls_cert_days
+
+    # Regex used to validate target hosts for mTLS proxying (SSRF protection)
+    # @return [String] host regex pattern
+    attr_accessor :pun_mtls_host_regex
+
+    # DNS resolver address(es) for nginx to use with variable-based proxy_pass.
+    # Auto-detected from /etc/resolv.conf if not set.
+    # @return [String] resolver address(es)
+    attr_accessor :pun_mtls_resolver
+
     #
     # Configuration module
     #
@@ -528,6 +549,19 @@ def set_default_configuration
       self.disable_bundle_user_config = true
       self.nginx_file_upload_max = '10737420000'
 
+      self.pun_mtls_enabled    = false
+      self.pun_mtls_pki_dir    = '~/ondemand/pki'
+      self.pun_mtls_cert_days  = 3650
+      self.pun_mtls_host_regex = '[\w.-]+'
+      self.pun_mtls_resolver   = begin
+        nameserver = File.readlines('/etc/resolv.conf')
+                         .grep(/^\s*nameserver\s+/)
+                         .first&.split&.last
+        nameserver || '127.0.0.1'
+      rescue
+        '127.0.0.1'
+      end
+
       read_configuration(default_config_path) if File.file?(default_config_path)
     end
 

nginx_stage/lib/nginx_stage/generators/pun_config_generator.rb

@@ -105,6 +105,79 @@ class PunConfigGenerator < Generator
       end
     end
 
+    # Generate per-user mTLS keypair if enabled and not already present or expired.
+    # Runs file operations in a forked child that drops privileges to the target
+    # user so that root_squash NFS shares are handled correctly.
+    add_hook :create_mtls_keypair do
+      if NginxStage.pun_mtls_enabled
+        require 'openssl'
+        require 'fileutils'
+
+        pki_dir = NginxStage.mtls_pki_dir(user: user)
+        cert_path = File.join(pki_dir, 'client.crt')
+        key_path  = File.join(pki_dir, 'client.key')
+        ca_path   = File.join(pki_dir, 'ca.crt')
+
+        # Check if cert exists and is not expired
+        needs_generation = true
+        if File.file?(cert_path) && File.file?(key_path)
+          begin
+            existing_cert = OpenSSL::X509::Certificate.new(File.read(cert_path))
+            needs_generation = false if existing_cert.not_after > Time.now
+          rescue OpenSSL::X509::CertificateError
+            # Corrupt cert, regenerate
+          end
+        end
+
+        if needs_generation
+          # Generate key material in the parent (no filesystem access needed)
+          key = OpenSSL::PKey::EC.generate('prime256v1')
+
+          cert = OpenSSL::X509::Certificate.new
+          cert.version = 2
+          cert.serial = OpenSSL::BN.rand(128)
+          cert.subject = OpenSSL::X509::Name.new([['CN', "#{user}@ondemand"]])
+          cert.issuer = cert.subject
+          cert.public_key = key
+          cert.not_before = Time.now - 60
+          cert.not_after = Time.now + (NginxStage.pun_mtls_cert_days * 86400)
+
+          ef = OpenSSL::X509::ExtensionFactory.new
+          ef.subject_certificate = cert
+          ef.issuer_certificate = cert
+          cert.add_extension(ef.create_extension('basicConstraints', 'CA:TRUE', true))
+          cert.add_extension(ef.create_extension('subjectKeyIdentifier', 'hash'))
+          cert.add_extension(ef.create_extension('keyUsage', 'digitalSignature,keyCertSign', true))
+
+          cert.sign(key, OpenSSL::Digest::SHA256.new)
+
+          key_pem  = key.to_pem
+          cert_pem = cert.to_pem
+
+          # Fork a child that drops to the user's UID/GID before writing files,
+          # so NFS root_squash shares see the real user, not root/nobody.
+          pw = Etc.getpwnam(user.to_s)
+          pid = Process.fork do
+            Process::GID.change_privilege(pw.gid)
+            Process::UID.change_privilege(pw.uid)
+
+            FileUtils.mkdir_p(pki_dir, mode: 0700)
+
+            File.write(key_path, key_pem)
+            File.chmod(0600, key_path)
+
+            File.write(cert_path, cert_pem)
+            File.chmod(0644, cert_path)
+
+            FileUtils.cp(cert_path, ca_path)
+            File.chmod(0644, ca_path)
+          end
+          _, status = Process.waitpid2(pid)
+          $stderr.puts "WARNING: mTLS keypair generation failed for #{user}" unless status.success?
+        end
+      end
+    end
+
     # Run the pre hook command. This eats the output and doesn't affect
     # the overall status of the PUN startup
     # This must come before anything that cleans the process environment

nginx_stage/lib/nginx_stage/views/pun_config_view.rb

@@ -145,6 +145,70 @@ def disable_bundle_user_config?
       NginxStage.disable_bundle_user_config
     end
 
+    # Whether mTLS proxying to compute nodes is enabled
+    # @return [Boolean] mTLS proxy enabled
+    def mtls_enabled?
+      NginxStage.pun_mtls_enabled
+    end
+
+    # Path to the mTLS client certificate
+    # @return [String] path to client cert
+    def mtls_client_cert_path
+      File.join(NginxStage.mtls_pki_dir(user: user), 'client.crt')
+    end
+
+    # Path to the mTLS client private key
+    # @return [String] path to client key
+    def mtls_client_key_path
+      File.join(NginxStage.mtls_pki_dir(user: user), 'client.key')
+    end
+
+    # Path to the mTLS CA certificate
+    # @return [String] path to CA cert
+    def mtls_ca_cert_path
+      File.join(NginxStage.mtls_pki_dir(user: user), 'ca.crt')
+    end
+
+    # Regex pattern for allowed mTLS proxy target hosts
+    # @return [String] host regex
+    def mtls_host_regex
+      NginxStage.pun_mtls_host_regex
+    end
+
+    # DNS resolver for nginx variable-based proxy_pass
+    # @return [String] resolver address(es)
+    def mtls_resolver
+      NginxStage.pun_mtls_resolver
+    end
+
+    # Expected CN of the upstream server's certificate for SSL name verification
+    # Headers to strip when proxying to compute nodes
+    # @return [Array<String>] list of header names
+    def strip_proxy_headers
+      %w[
+        Authorization
+        X-Forwarded-User
+        OIDC_CLAIM_sub
+        OIDC_CLAIM_preferred_username
+        OIDC_CLAIM_given_name
+        OIDC_CLAIM_zoneinfo
+        OIDC_CLAIM_locale
+        OIDC_CLAIM_email
+        OIDC_CLAIM_email_verified
+        OIDC_CLAIM_iss
+        OIDC_CLAIM_nonce
+        OIDC_CLAIM_aud
+        OIDC_CLAIM_acr
+        OIDC_CLAIM_azp
+        OIDC_CLAIM_auth_time
+        OIDC_CLAIM_exp
+        OIDC_CLAIM_iat
+        OIDC_CLAIM_jti
+        OIDC_access_token
+        OIDC_access_token_expires
+      ]
+    end
+
     # View used to confirm whether the user wants to restart the PUN to reload
     # configuration changes
     # @return [String] restart confirmation view

nginx_stage/share/nginx_stage_example.yml

@@ -52,6 +52,22 @@
 # error page can be customized with this mechanism.
 #pun_custom_html_root: '/etc/ood/config/pun/html'
 
+# Whether mTLS proxying from the PUN to compute nodes is enabled
+#pun_mtls_enabled: false
+
+# Directory under the user's home for storing mTLS PKI material
+#pun_mtls_pki_dir: '~/ondemand/pki'
+
+# Validity period in days for the generated mTLS certificate
+#pun_mtls_cert_days: 3650
+
+# Regex used to validate target hosts for mTLS proxying (SSRF protection)
+#pun_mtls_host_regex: '[\w.-]+'
+
+# DNS resolver for nginx to use with variable-based proxy_pass
+# Auto-detected from /etc/resolv.conf if not set
+#pun_mtls_resolver: '8.8.8.8'
+
 # Location of the ERB templates used in the generation of the NGINX configs
 #
 #template_root: '/opt/ood/nginx_stage/templates'

nginx_stage/templates/pun.conf.erb

@@ -78,6 +78,14 @@ http {
 
   default_type application/octet-stream;
 
+  <%- if mtls_enabled? -%>
+  # WebSocket upgrade map for mTLS proxy
+  map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+  }
+  <%- end -%>
+
   log_format   main '<%= log_format %>';
 
   access_log   <%= access_log_path %> main;
@@ -132,5 +140,96 @@ http {
     <%- app_configs.each do |app_config| -%>
     include <%= app_config %>;
     <%- end -%>
+
+    <%- if mtls_enabled? -%>
+    # mTLS proxy to compute node services (full path preserved)
+    location ~ ^/pun/proxy/(?<target_host><%= mtls_host_regex %>)/(?<target_port>\d+)(?<target_uri>/.*)?$ {
+
+      # Resolver needed for variable-based proxy_pass
+      resolver <%= mtls_resolver %> valid=300s;
+      resolver_timeout 5s;
+
+      # Strip sensitive headers before proxying
+      <%- strip_proxy_headers.each do |header| -%>
+      proxy_set_header <%= header %> "";
+      <%- end -%>
+
+      # Forward useful context headers
+      proxy_set_header Host $target_host:$target_port;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Real-IP $remote_addr;
+
+      # WebSocket support
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection $connection_upgrade;
+      proxy_http_version 1.1;
+
+      # mTLS client certificate configuration
+      proxy_ssl_certificate      <%= mtls_client_cert_path %>;
+      proxy_ssl_certificate_key  <%= mtls_client_key_path %>;
+      proxy_ssl_trusted_certificate <%= mtls_ca_cert_path %>;
+      proxy_ssl_verify on;
+      proxy_ssl_verify_depth 1;
+      proxy_ssl_server_name on;
+      proxy_ssl_name $target_host;
+      proxy_ssl_protocols TLSv1.2 TLSv1.3;
+
+      # URL rewriting for Location headers
+      proxy_redirect https://$target_host:$target_port/ /pun/proxy/$target_host/$target_port/;
+
+      # Rewrite Set-Cookie domain and path
+      proxy_cookie_domain $target_host localhost;
+      proxy_cookie_path / /pun/proxy/$target_host/$target_port/;
+
+      # Preserve the full request path including /pun/proxy/<host>/<port>
+      proxy_pass https://$target_host:$target_port$target_url$request_uri;
+    }
+
+    # mTLS relative proxy to compute node services (path prefix stripped)
+    location ~ ^/pun/rproxy/(?<target_host><%= mtls_host_regex %>)/(?<target_port>\d+)(?<target_uri>/.*)?$ {
+
+      # Resolver needed for variable-based proxy_pass
+      resolver <%= mtls_resolver %> valid=300s;
+      resolver_timeout 5s;
+
+      # Strip sensitive headers before proxying
+      <%- strip_proxy_headers.each do |header| -%>
+      proxy_set_header <%= header %> "";
+      <%- end -%>
+
+      # Forward useful context headers
+      proxy_set_header Host $target_host:$target_port;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header X-Forwarded-Prefix /pun/rproxy/$target_host/$target_port;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Real-IP $remote_addr;
+
+      # WebSocket support
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection $connection_upgrade;
+      proxy_http_version 1.1;
+
+      # mTLS client certificate configuration
+      proxy_ssl_certificate      <%= mtls_client_cert_path %>;
+      proxy_ssl_certificate_key  <%= mtls_client_key_path %>;
+      proxy_ssl_trusted_certificate <%= mtls_ca_cert_path %>;
+      proxy_ssl_verify on;
+      proxy_ssl_verify_depth 1;
+      proxy_ssl_server_name on;
+      proxy_ssl_name $target_host;
+      proxy_ssl_protocols TLSv1.2 TLSv1.3;
+
+      # URL rewriting for Location headers
+      proxy_redirect https://$target_host:$target_port/ /pun/rproxy/$target_host/$target_port/;
+
+      # Rewrite Set-Cookie domain and path
+      proxy_cookie_domain $target_host localhost;
+      proxy_cookie_path / /pun/rproxy/$target_host/$target_port/;
+
+      # Proxy to compute node over TLS
+      proxy_pass https://$target_host:$target_port$target_url$target_uri$is_args$args;
+    }
+    <%- end -%>
   }
 }

ood-portal-generator/lib/ood_portal_generator/view.rb

@@ -96,6 +96,10 @@ def initialize(opts = {})
       @pun_pre_hook_exports   = opts.fetch(:pun_pre_hook_exports, nil)
       @pun_pre_hook_root_cmd  = opts.fetch(:pun_pre_hook_root_cmd, nil)
 
+      # mTLS proxy sub-uri (header stripping at Apache layer)
+      @pun_proxy_uri          = opts.fetch(:pun_proxy_uri, nil)
+      @pun_rproxy_uri         = opts.fetch(:pun_rproxy_uri, @pun_proxy_uri ? "/pun/rproxy" : nil)
+
       # OpenID Connect sub-uri
       @oidc_uri           = opts.fetch(:oidc_uri, nil)
       @oidc_discover_uri  = opts.fetch(:oidc_discover_uri, nil)

ood-portal-generator/share/ood_portal_example.yml

@@ -337,6 +337,21 @@
 # Default: null (pass nothing)
 #pun_pre_hook_exports: null
 
+# Sub-uri for mTLS proxy requests. When set, Apache strips sensitive headers
+# (Authorization, OIDC claims, session cookies) from requests to this path
+# before they reach the PUN.
+# Example:
+#     pun_proxy_uri: '/pun/proxy'
+# Default: null (disabled)
+#pun_proxy_uri: null
+
+# Sub-uri for mTLS relative proxy requests. This behaves like rnode/rproxy by
+# stripping the /pun/rproxy/<host>/<port> prefix before forwarding upstream.
+# Example:
+#     pun_rproxy_uri: '/pun/rproxy'
+# Default: '/pun/rproxy' when pun_proxy_uri is set, otherwise null
+#pun_rproxy_uri: null
+
 #
 # Support for OpenID Connect
 #