error instead of panicing when marshaling hash objects

Author: qmuntal

hash.go

@@ -10,6 +10,7 @@ import (
 	"hash"
 	"runtime"
 	"strconv"
+	"sync"
 	"unsafe"
 )
 
@@ -110,6 +111,37 @@ func SHA3_512(p []byte) (sum [64]byte) {
 	return
 }
 
+var isMarshallableCache sync.Map
+
+// isHashMarshallable returns true if the memory layout of cb
+// is known by this library and can therefore be marshalled.
+func isHashMarshallable(ch crypto.Hash) bool {
+	if vMajor == 1 {
+		return true
+	}
+	if v, ok := isMarshallableCache.Load(ch); ok {
+		return v.(bool)
+	}
+	md := cryptoHashToMD(ch)
+	if md == nil {
+		return false
+	}
+	prov := C.go_openssl_EVP_MD_get0_provider(md)
+	if prov == nil {
+		return false
+	}
+	cname := C.go_openssl_OSSL_PROVIDER_get0_name(prov)
+	if cname == nil {
+		return false
+	}
+	name := C.GoString(cname)
+	// We only know the memory layout of the built-in providers.
+	// See evpHash.hashState for more details.
+	marshallable := name == "default" || name == "fips"
+	isMarshallableCache.Store(ch, marshallable)
+	return marshallable
+}
+
 // evpHash implements generic hash methods.
 type evpHash struct {
 	ctx C.GO_EVP_MD_CTX_PTR
@@ -119,6 +151,8 @@ type evpHash struct {
 	ctx2      C.GO_EVP_MD_CTX_PTR
 	size      int
 	blockSize int
+
+	marshallable bool
 }
 
 func newEvpHash(ch crypto.Hash, size, blockSize int) *evpHash {
@@ -137,6 +171,8 @@ func newEvpHash(ch crypto.Hash, size, blockSize int) *evpHash {
 		ctx2:      ctx2,
 		size:      size,
 		blockSize: blockSize,
+
+		marshallable: isHashMarshallable(ch),
 	}
 	runtime.SetFinalizer(h, (*evpHash).finalize)
 	return h
@@ -200,6 +236,9 @@ func (h *evpHash) sum(out []byte) {
 // The EVP_MD_CTX memory layout has changed in OpenSSL 3
 // and the property holding the internal structure is no longer md_data but algctx.
 func (h *evpHash) hashState() unsafe.Pointer {
+	if !h.marshallable {
+		return nil
+	}
 	switch vMajor {
 	case 1:
 		// https://github.com/openssl/openssl/blob/0418e993c717a6863f206feaa40673a261de7395/crypto/evp/evp_local.h#L12.

openssl_test.go

@@ -54,7 +54,7 @@ func TestMain(m *testing.M) {
 		// or that there is a bug in the Init code.
 		panic(err)
 	}
-	_ = openssl.SetFIPS(true) // Skip the error as we still want to run the tests on machines without FIPS support.
+	//_ = openssl.SetFIPS(true) // Skip the error as we still want to run the tests on machines without FIPS support.
 	fmt.Println("OpenSSL version:", openssl.VersionText())
 	fmt.Println("FIPS enabled:", openssl.FIPS())
 	status := m.Run()

shims.h

@@ -190,9 +190,11 @@ DEFINEFUNC_3_0(int, EVP_default_properties_is_fips_enabled, (GO_OSSL_LIB_CTX_PTR
 DEFINEFUNC_3_0(int, EVP_default_properties_enable_fips, (GO_OSSL_LIB_CTX_PTR libctx, int enable), (libctx, enable)) \
 DEFINEFUNC_3_0(int, OSSL_PROVIDER_available, (GO_OSSL_LIB_CTX_PTR libctx, const char *name), (libctx, name)) \
 DEFINEFUNC_3_0(GO_OSSL_PROVIDER_PTR, OSSL_PROVIDER_load, (GO_OSSL_LIB_CTX_PTR libctx, const char *name), (libctx, name)) \
+DEFINEFUNC_3_0(const char *, OSSL_PROVIDER_get0_name, (const GO_OSSL_PROVIDER_PTR prov), (prov)) \
 DEFINEFUNC_3_0(GO_EVP_MD_PTR, EVP_MD_fetch, (GO_OSSL_LIB_CTX_PTR ctx, const char *algorithm, const char *properties), (ctx, algorithm, properties)) \
 DEFINEFUNC_3_0(void, EVP_MD_free, (GO_EVP_MD_PTR md), (md)) \
 DEFINEFUNC_3_0(const char *, EVP_MD_get0_name, (const GO_EVP_MD_PTR md), (md)) \
+DEFINEFUNC_3_0(const GO_OSSL_PROVIDER_PTR, EVP_MD_get0_provider, (const GO_EVP_MD_PTR md), (md)) \
 DEFINEFUNC(int, RAND_bytes, (unsigned char *arg0, int arg1), (arg0, arg1)) \
 DEFINEFUNC_RENAMED_1_1(GO_EVP_MD_CTX_PTR, EVP_MD_CTX_new, EVP_MD_CTX_create, (void), ()) \
 DEFINEFUNC_RENAMED_1_1(void, EVP_MD_CTX_free, EVP_MD_CTX_destroy, (GO_EVP_MD_CTX_PTR ctx), (ctx)) \