diff --git a/ext/openssl/ossl_provider.c b/ext/openssl/ossl_provider.c
index 529a5e1c7..65c978435 100644
--- a/ext/openssl/ossl_provider.c
+++ b/ext/openssl/ossl_provider.c
@@ -57,11 +57,11 @@ ossl_provider_s_load(VALUE klass, VALUE name)
 
     const char *provider_name_ptr = StringValueCStr(name);
 
+    obj = NewProvider(klass);
     provider = OSSL_PROVIDER_load(NULL, provider_name_ptr);
     if (provider == NULL) {
         ossl_raise(eProviderError, "Failed to load %s provider", provider_name_ptr);
     }
-    obj = NewProvider(klass);
     SetProvider(obj, provider);
 
     return obj;
@@ -182,6 +182,32 @@ ossl_provider_inspect(VALUE self)
                       rb_obj_class(self), OSSL_PROVIDER_get0_name(prov));
 }
 
+#if OSSL_OPENSSL_PREREQ(3, 5, 0)
+/*
+ * call-seq:
+ *    provider.add_conf_parameter(name, value) -> nil
+ *
+ * Sets the provider configuration parameter _name_ to _value_. Both name and
+ * value must be given as strings.
+ *
+ * See the documentation of the provider for possible parameters. See also the
+ * man page OSSL_PROVIDER_add_conf_parameter(3).
+ *
+ * This requires OpenSSL 3.5.0 or later.
+ */
+static VALUE
+ossl_provider_add_conf_parameter(VALUE self, VALUE name, VALUE value)
+{
+    OSSL_PROVIDER *prov;
+
+    GetProvider(self, prov);
+    if (OSSL_PROVIDER_add_conf_parameter(prov, StringValueCStr(name),
+                                         StringValueCStr(value)) != 1)
+        ossl_raise(eProviderError, "OSSL_PROVIDER_add_conf_parameter");
+    return Qnil;
+}
+#endif
+
 void
 Init_ossl_provider(void)
 {
@@ -200,6 +226,9 @@ Init_ossl_provider(void)
     rb_define_method(cProvider, "unload", ossl_provider_unload, 0);
     rb_define_method(cProvider, "name", ossl_provider_get_name, 0);
     rb_define_method(cProvider, "inspect", ossl_provider_inspect, 0);
+#if OSSL_OPENSSL_PREREQ(3, 5, 0)
+    rb_define_method(cProvider, "add_conf_parameter", ossl_provider_add_conf_parameter, 2);
+#endif
 }
 #else
 void
diff --git a/test/openssl/test_provider.rb b/test/openssl/test_provider.rb
index 6f85c00c9..c7071f5f5 100644
--- a/test/openssl/test_provider.rb
+++ b/test/openssl/test_provider.rb
@@ -70,6 +70,25 @@ def test_openssl_legacy_provider
     end;
   end
 
+  def test_add_conf_parameter
+    with_openssl <<-'end;'
+      prov = OpenSSL::Provider.load("null")
+      assert_raise(TypeError) { prov.add_conf_parameter("foo", nil) }
+
+      # This assumes that ML-DSA is provided by the "default" provider, which
+      # may not always be the case.
+      # TODO: OpenSSL::PKey should allow specifying the provider to use
+      return if OpenSSL.fips_mode
+      pkey = OpenSSL::PKey.generate_key("ML-DSA-44")
+      out_a = pkey.private_to_der
+      prov = OpenSSL::Provider.load("default")
+      prov.add_conf_parameter("ml-dsa.output_formats", "seed-only")
+      out_b = pkey.private_to_der
+      omit "ML-DSA not provided by the \"default\" provider?" if out_a == out_b
+      assert_not_equal(out_a, out_b)
+    end;
+  end if openssl?(3, 5, 0)
+
   private
 
   # this is required because OpenSSL::Provider methods change global state