[ruby/openssl] pkey: support 'one-shot' signing and verification

OpenSSL 1.1.1 added EVP_DigestSign() and EVP_DigestVerify() functions
to the interface. Some EVP_PKEY methods such as PureEdDSA algorithms
do not support the streaming mechanism and require us to use them.

https://github.com/ruby/openssl/commit/ae19454592

ext/openssl/ossl_pkey.c

@@ -771,6 +771,26 @@ ossl_pkey_sign(VALUE self, VALUE digest, VALUE data)
         EVP_MD_CTX_free(ctx);
         ossl_raise(ePKeyError, "EVP_DigestSignInit");
     }
+#if OPENSSL_VERSION_NUMBER >= 0x10101000 && !defined(LIBRESSL_VERSION_NUMBER)
+    if (EVP_DigestSign(ctx, NULL, &siglen, (unsigned char *)RSTRING_PTR(data),
+                       RSTRING_LEN(data)) < 1) {
+        EVP_MD_CTX_free(ctx);
+        ossl_raise(ePKeyError, "EVP_DigestSign");
+    }
+    if (siglen > LONG_MAX)
+        rb_raise(ePKeyError, "signature would be too large");
+    sig = ossl_str_new(NULL, (long)siglen, &state);
+    if (state) {
+        EVP_MD_CTX_free(ctx);
+        rb_jump_tag(state);
+    }
+    if (EVP_DigestSign(ctx, (unsigned char *)RSTRING_PTR(sig), &siglen,
+                       (unsigned char *)RSTRING_PTR(data),
+                       RSTRING_LEN(data)) < 1) {
+        EVP_MD_CTX_free(ctx);
+        ossl_raise(ePKeyError, "EVP_DigestSign");
+    }
+#else
     if (EVP_DigestSignUpdate(ctx, RSTRING_PTR(data), RSTRING_LEN(data)) < 1) {
         EVP_MD_CTX_free(ctx);
         ossl_raise(ePKeyError, "EVP_DigestSignUpdate");
@@ -791,6 +811,7 @@ ossl_pkey_sign(VALUE self, VALUE digest, VALUE data)
         EVP_MD_CTX_free(ctx);
         ossl_raise(ePKeyError, "EVP_DigestSignFinal");
     }
+#endif
     EVP_MD_CTX_free(ctx);
     rb_str_set_len(sig, siglen);
     return sig;
@@ -839,6 +860,14 @@ ossl_pkey_verify(VALUE self, VALUE digest, VALUE sig, VALUE data)
         EVP_MD_CTX_free(ctx);
         ossl_raise(ePKeyError, "EVP_DigestVerifyInit");
     }
+#if OPENSSL_VERSION_NUMBER >= 0x10101000 && !defined(LIBRESSL_VERSION_NUMBER)
+    ret = EVP_DigestVerify(ctx, (unsigned char *)RSTRING_PTR(sig),
+                           RSTRING_LEN(sig), (unsigned char *)RSTRING_PTR(data),
+                           RSTRING_LEN(data));
+    EVP_MD_CTX_free(ctx);
+    if (ret < 0)
+        ossl_raise(ePKeyError, "EVP_DigestVerify");
+#else
     if (EVP_DigestVerifyUpdate(ctx, RSTRING_PTR(data), RSTRING_LEN(data)) < 1) {
         EVP_MD_CTX_free(ctx);
         ossl_raise(ePKeyError, "EVP_DigestVerifyUpdate");
@@ -848,6 +877,7 @@ ossl_pkey_verify(VALUE self, VALUE digest, VALUE sig, VALUE data)
     EVP_MD_CTX_free(ctx);
     if (ret < 0)
         ossl_raise(ePKeyError, "EVP_DigestVerifyFinal");
+#endif
     if (ret)
         return Qtrue;
     else {

test/openssl/test_pkey.rb

@@ -80,4 +80,49 @@ class OpenSSL::TestPKey < OpenSSL::PKeyTestCase
       pkey.verify("SHA256", "data", hmac)
     }
   end
+
+  def test_ed25519
+    # Test vector from RFC 8032 Section 7.1 TEST 2
+    priv_pem = <<~EOF
+    -----BEGIN PRIVATE KEY-----
+    MC4CAQAwBQYDK2VwBCIEIEzNCJso/5banbbDRuwRTg9bijGfNaumJNqM9u1PuKb7
+    -----END PRIVATE KEY-----
+    EOF
+    pub_pem = <<~EOF
+    -----BEGIN PUBLIC KEY-----
+    MCowBQYDK2VwAyEAPUAXw+hDiVqStwqnTRt+vJyYLM8uxJaMwM1V8Sr0Zgw=
+    -----END PUBLIC KEY-----
+    EOF
+    begin
+      priv = OpenSSL::PKey.read(priv_pem)
+      pub = OpenSSL::PKey.read(pub_pem)
+    rescue OpenSSL::PKey::PKeyError
+      # OpenSSL < 1.1.1
+      pend "Ed25519 is not implemented"
+    end
+    assert_instance_of OpenSSL::PKey::PKey, priv
+    assert_instance_of OpenSSL::PKey::PKey, pub
+    assert_equal priv_pem, priv.private_to_pem
+    assert_equal pub_pem, priv.public_to_pem
+    assert_equal pub_pem, pub.public_to_pem
+
+    sig = [<<~EOF.gsub(/[^0-9a-f]/, "")].pack("H*")
+    92a009a9f0d4cab8720e820b5f642540
+    a2b27b5416503f8fb3762223ebdb69da
+    085ac1e43e15996e458f3613d0f11d8c
+    387b2eaeb4302aeeb00d291612bb0c00
+    EOF
+    data = ["72"].pack("H*")
+    assert_equal sig, priv.sign(nil, data)
+    assert_equal true, priv.verify(nil, sig, data)
+    assert_equal true, pub.verify(nil, sig, data)
+    assert_equal false, pub.verify(nil, sig, data.succ)
+
+    # PureEdDSA wants nil as the message digest
+    assert_raise(OpenSSL::PKey::PKeyError) { priv.sign("SHA512", data) }
+    assert_raise(OpenSSL::PKey::PKeyError) { pub.verify("SHA512", sig, data) }
+
+    # Ed25519 pkey type does not support key derivation
+    assert_raise(OpenSSL::PKey::PKeyError) { priv.derive(pub) }
+  end
 end