diff --git a/pkg/client/rp/jwks.go b/pkg/client/rp/jwks.go
index 3fb26c5..e7b240c 100644
--- a/pkg/client/rp/jwks.go
+++ b/pkg/client/rp/jwks.go
@@ -88,26 +88,31 @@ func (r *remoteKeySet) VerifySignature(ctx context.Context, jws *jose.JSONWebSig
 	if alg == "" {
 		alg = r.defaultAlg
 	}
-	keys := r.keysFromCache()
-	if len(keys) == 0 {
-		return r.verifySignatureRemote(ctx, jws, keyID, alg)
-	}
-	payload, err := r.verifySignatureCached(jws, keys, keyID, alg)
+	payload, err := r.verifySignatureCached(jws, keyID, alg)
 	if payload != nil {
 		return payload, nil
 	}
 	if err != nil && keyID != "" || r.skipRemoteCheck {
-		return nil, fmt.Errorf("invalid signature: %w", err)
+		return nil, err
 	}
 	return r.verifySignatureRemote(ctx, jws, keyID, alg)
 }
 
-func (r *remoteKeySet) verifySignatureCached(jws *jose.JSONWebSignature, keys []jose.JSONWebKey, keyID, alg string) ([]byte, error) {
+func (r *remoteKeySet) verifySignatureCached(jws *jose.JSONWebSignature, keyID, alg string) ([]byte, error) {
+	keys := r.keysFromCache()
+	if len(keys) == 0 {
+		return nil, nil
+	}
 	key, err := oidc.FindMatchingKey(keyID, oidc.KeyUseSignature, alg, keys...)
 	if err != nil {
-		return nil, err
+		//ignore error here, try with remote keys
+		return nil, nil //nolint:nilerr
 	}
-	return jws.Verify(&key)
+	payload, err := jws.Verify(&key)
+	if err != nil {
+		return nil, fmt.Errorf("signature verification failed: %w", err)
+	}
+	return payload, nil
 }
 
 func (r *remoteKeySet) verifySignatureRemote(ctx context.Context, jws *jose.JSONWebSignature, keyID, alg string) ([]byte, error) {
@@ -119,7 +124,11 @@ func (r *remoteKeySet) verifySignatureRemote(ctx context.Context, jws *jose.JSON
 	if err != nil {
 		return nil, fmt.Errorf("unable to validate signature: %w", err)
 	}
-	return jws.Verify(&key)
+	payload, err := jws.Verify(&key)
+	if err != nil {
+		return nil, fmt.Errorf("signature verification failed: %w", err)
+	}
+	return payload, nil
 }
 
 func (r *remoteKeySet) keysFromCache() (keys []jose.JSONWebKey) {
diff --git a/pkg/oidc/verifier.go b/pkg/oidc/verifier.go
index f8470b5..4284d17 100644
--- a/pkg/oidc/verifier.go
+++ b/pkg/oidc/verifier.go
@@ -39,6 +39,7 @@ var (
 	ErrSignatureMultiple       = errors.New("id_token contains multiple signatures")
 	ErrSignatureUnsupportedAlg = errors.New("signature algorithm not supported")
 	ErrSignatureInvalidPayload = errors.New("signature does not match Payload")
+	ErrSignatureInvalid        = errors.New("invalid signature")
 	ErrExpired                 = errors.New("token has expired")
 	ErrIatMissing              = errors.New("issuedAt of token is missing")
 	ErrIatInFuture             = errors.New("issuedAt of token is in the future")
@@ -143,7 +144,7 @@ func CheckSignature(ctx context.Context, token string, payload []byte, claims Cl
 
 	signedPayload, err := set.VerifySignature(ctx, jws)
 	if err != nil {
-		return err
+		return fmt.Errorf("%w (%v)", ErrSignatureInvalid, err)
 	}
 
 	if !bytes.Equal(signedPayload, payload) {