pkey: add :format keyword argument to PKey::PKey#to_{der,pem}

Currently they return XXXPrivateKey format (PKCS ruby#1 RSAPrivateKey for
RSA, OpenSSL format for DSA, anad SEC 1 ECPrivateKey for EC) when the
instance contains the private key, or SubjectPublicKeyInfo) otherwise.
There is no option to force the format, so users need to duplicate the
key first and remove the private key if they want SubjectPublicKeyInfo
DER from a key.

This resolves it by adding :format keyword argument to
PKey::PKey#to_{der,pem}. Users can pass either :*PrivateKey or
:SubjectPublicKeyInfo.

Author: rhenium

ext/openssl/ossl_pkey.c

@@ -190,35 +190,83 @@ DupPrivPKeyPtr(VALUE obj)
     return pkey;
 }
 
+static enum ossl_pkey_export_format
+export_format(VALUE obj, int type, VALUE opts)
+{
+    ID kwids[1], id;
+    VALUE kwargs[1];
+
+    kwids[0] = rb_intern("format");
+    rb_get_kwargs(opts, kwids, 0, 1, kwargs);
+    if (kwargs[0] == Qundef) {
+	if (RTEST(rb_funcall(obj, id_private_q, 0))) /* XXX */
+	    return OSSL_PKEY_PRIVKEY;
+	else
+	    return OSSL_PKEY_PUBKEY;
+    }
+
+    id = SYM2ID(kwargs[0]);
+    if (id == rb_intern("SubjectPublicKeyInfo")) {
+	return OSSL_PKEY_PUBKEY;
+    }
+    else {
+	if (!RTEST(rb_funcall(obj, id_private_q, 0))) /* XXX */
+	    ossl_raise(ePKeyError, "private key required");
+	if (type == EVP_PKEY_RSA && id == rb_intern("RSAPrivateKey") ||
+	    type == EVP_PKEY_DSA && id == rb_intern("DSAPrivateKey") ||
+#if !defined(OPENSSL_NO_EC)
+	    type == EVP_PKEY_EC  && id == rb_intern("ECPrivateKey") ||
+#endif
+	    0) {
+	    return OSSL_PKEY_PRIVKEY;
+	}
+	else {
+	    ossl_raise(rb_eArgError, "unsupported format %"PRIsVALUE, kwargs[0]);
+	}
+    }
+}
+
 /*
  * call-seq:
- *   key.export([cipher, pass_phrase]) -> String
- *   key.to_pem([cipher, pass_phrase]) -> String
- *   key.to_s([cipher, pass_phrase])   -> String
+ *   key.export([cipher, pass_phrase], format: nil) -> String
+ *   key.to_pem([cipher, pass_phrase], format: nil) -> String
+ *   key.to_s([cipher, pass_phrase], format: nil)   -> String
  *
  * Encodes the PKey into a PEM string. If +cipher+ and +pass_phrase+ are given,
  * they will be used to encrypt the key. +cipher+ must be an OpenSSL::Cipher
  * instance. Note that encryption will only be effective for a private key,
- * public keys will always be encoded in unencrypted format.
+ * public keys will always be encoded in unencrypted format. The output format
+ * can be specified with +format+. It must be one of these:
+ *
+ * :SubjectPublicKeyInfo:: X.509 SubjectPublicKeyInfo format
+ * :RSAPrivateKey:: PKCS #1 RSAPrivateKey format
+ * :DSAPrivateKey:: OpenSSL DSAPrivateKey format
+ * :ECPrivateKey:: SEC 1 ECPrivateKey format
+ *
+ * When +format+ is not given, the default (:SubjectPublicKeyInfo for public
+ * keys, :*PrivateKey for private keys) is used.
  *
  * === Examples
  *  dsa = OpenSSL::PKey::DSA.new(1024) # generates a new key
- *  dsa.to_pem #=> "-----BEGIN DSA PRIVATE KEY-----\n..."
+ *  dsa.to_pem
+ *  #=> "-----BEGIN DSA PRIVATE KEY-----\n..."
+ *  dsa.to_pem(format: :SubjectPublicKeyInfo)
+ *  #=> "-----BEGIN PUBLIC KEY-----\n..."
  */
 static VALUE
 ossl_pkey_export(int argc, VALUE *argv, VALUE self)
 {
     EVP_PKEY *pkey;
     BIO *out;
     const EVP_CIPHER *ciph = NULL;
-    VALUE cipher, pass;
+    VALUE cipher, pass, opts;
     int ret, type;
     enum ossl_pkey_export_format format;
 
     GetPKey(self, pkey);
     type = EVP_PKEY_base_id(pkey);
-    rb_scan_args(argc, argv, "02", &cipher, &pass);
-    format = RTEST(rb_funcall(self, id_private_q, 0)) ? OSSL_PKEY_PRIVKEY : OSSL_PKEY_PUBKEY;
+    rb_scan_args(argc, argv, "02:", &cipher, &pass, &opts);
+    format = export_format(self, type, opts);
     if (!NIL_P(cipher)) {
 	ciph = GetCipherPtr(cipher);
 	pass = ossl_pem_passwd_value(pass);
@@ -265,21 +313,23 @@ ossl_pkey_export(int argc, VALUE *argv, VALUE self)
 
 /*
  * call-seq:
- *   key.to_der -> String
+ *   key.to_der(format: nil) -> String
  *
- * Encodes the PKey into a DER string.
+ * Encodes the PKey into a DER string. +format+ is handled in the same way as
+ * #export.
  */
 static VALUE
-ossl_pkey_to_der(VALUE self)
+ossl_pkey_to_der(int argc, VALUE *argv, VALUE self)
 {
     EVP_PKEY *pkey;
-    VALUE str;
+    VALUE opts, str;
     int type;
     enum ossl_pkey_export_format format;
 
     GetPKey(self, pkey);
     type = EVP_PKEY_base_id(pkey);
-    format = RTEST(rb_funcall(self, id_private_q, 0)) ? OSSL_PKEY_PRIVKEY : OSSL_PKEY_PUBKEY;
+    rb_scan_args(argc, argv, "0:", &opts);
+    format = export_format(self, type, opts);
 
     switch (format) {
       case OSSL_PKEY_PUBKEY:
@@ -537,7 +587,7 @@ Init_ossl_pkey(void)
     rb_define_method(cPKey, "export", ossl_pkey_export, -1);
     rb_define_alias(cPKey, "to_pem", "export");
     rb_define_alias(cPKey, "to_s", "export");
-    rb_define_method(cPKey, "to_der", ossl_pkey_to_der, 0);
+    rb_define_method(cPKey, "to_der", ossl_pkey_to_der, -1);
 
     id_private_q = rb_intern("private?");
 

test/test_pkey_dsa.rb

@@ -134,20 +134,31 @@ def test_to_der_DSAPrivateKey
     # defaults to DSAPrivateKey format when priv_key is present
     der = DSA512.to_der
     check_DSAPrivateKey(der, DSA512)
+
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      dup_public(DSA512).to_der(format: :DSAPrivateKey)
+    }
   end
 
   def test_to_der_DSA_PUBKEY
     dsa = dup_public(DSA512)
     # defaults to PUBKEY format when priv_key is absent
     der = dsa.to_der
     check_PUBKEY(der, DSA512)
+
+    der = DSA512.to_der(format: :SubjectPublicKeyInfo)
+    check_PUBKEY(der, DSA512)
   end
 
   def test_export_DSAPrivateKey
     pem = DSA512.to_pem
     header, *body, _ = pem.lines
     assert_equal("-----BEGIN DSA PRIVATE KEY-----\n", header)
     check_DSAPrivateKey(body.join.unpack("m")[0], DSA512)
+
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      dup_public(DSA512).to_pem(format: :DSAPrivateKey)
+    }
   end
 
   def test_export_DSA_PUBKEY
@@ -156,6 +167,11 @@ def test_export_DSA_PUBKEY
     header, *body, _ = pem.lines
     assert_equal("-----BEGIN PUBLIC KEY-----\n", header)
     check_PUBKEY(body.join.unpack("m")[0], DSA512)
+
+    pem = DSA512.to_pem(format: :SubjectPublicKeyInfo)
+    header, *body, _ = pem.lines
+    assert_equal("-----BEGIN PUBLIC KEY-----\n", header)
+    check_PUBKEY(body.join.unpack("m")[0], DSA512)
   end
 
   def test_dup

test/test_pkey_ec.rb

@@ -235,19 +235,30 @@ def test_read_EC_PUBKEY_pem
   def test_to_der_ECPrivateKey
     der = P256.to_der
     check_ECPrivateKey(der, P256)
+
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      dup_public(P256).to_der(format: :ECPrivateKey)
+    }
   end
 
   def test_to_der_EC_PUBKEY
     ec = dup_public(P256)
     der = ec.to_der
     check_PUBKEY(der, P256)
+
+    der = P256.to_der(format: :SubjectPublicKeyInfo)
+    check_PUBKEY(der, P256)
   end
 
   def test_export_ECPrivateKey
     pem = P256.to_pem
     header, *body, _ = pem.lines
     assert_equal("-----BEGIN EC PRIVATE KEY-----\n", header)
     check_ECPrivateKey(body.join.unpack("m")[0], P256)
+
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      dup_public(P256).to_pem(format: :ECPrivateKey)
+    }
   end
 
   def test_export_EC_PUBKEY
@@ -256,6 +267,11 @@ def test_export_EC_PUBKEY
     header, *body, _ = pem.lines
     assert_equal("-----BEGIN PUBLIC KEY-----\n", header)
     check_PUBKEY(body.join.unpack("m")[0], P256)
+
+    pem = P256.to_pem(format: :SubjectPublicKeyInfo)
+    header, *body, _ = pem.lines
+    assert_equal("-----BEGIN PUBLIC KEY-----\n", header)
+    check_PUBKEY(body.join.unpack("m")[0], P256)
   end
 
   def test_ec_point_mul

test/test_pkey_rsa.rb

@@ -222,20 +222,31 @@ def test_to_der_RSAPrivateKey
     # defaults to RSAPrivateKey format when private components are present
     der = RSA1024.to_der
     check_RSAPrivateKey(der, RSA1024)
+
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      dup_public(RSA1024).to_der(format: :RSAPrivateKey)
+    }
   end
 
   def test_to_der_RSA_PUBKEY
     rsa = dup_public(RSA1024)
     # defaults to PUBKEY format when private components are absent
     der = rsa.to_der
     check_PUBKEY(der, RSA1024)
+
+    der = RSA1024.to_der(format: :SubjectPublicKeyInfo)
+    check_PUBKEY(der, RSA1024)
   end
 
   def test_export_RSAPrivateKey
     pem = RSA1024.to_pem
     header, *body, _ = pem.lines
     assert_equal("-----BEGIN RSA PRIVATE KEY-----\n", header)
     check_RSAPrivateKey(body.join.unpack("m")[0], RSA1024)
+
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      dup_public(RSA1024).to_pem(format: :RSAPrivateKey)
+    }
   end
 
   def test_export_RSA_PUBKEY
@@ -244,6 +255,11 @@ def test_export_RSA_PUBKEY
     header, *body, _ = pem.lines
     assert_equal("-----BEGIN PUBLIC KEY-----\n", header)
     check_PUBKEY(body.join.unpack("m")[0], RSA1024)
+
+    pem = RSA1024.to_pem(format: :SubjectPublicKeyInfo)
+    header, *body, _ = pem.lines
+    assert_equal("-----BEGIN PUBLIC KEY-----\n", header)
+    check_PUBKEY(body.join.unpack("m")[0], RSA1024)
   end
 
   def test_dup