diff --git a/agent/findkey.c b/agent/findkey.c
index fa9e5b548..eec85ba67 100644
--- a/agent/findkey.c
+++ b/agent/findkey.c
@@ -996,7 +996,10 @@ agent_key_from_file (ctrl_t ctrl, const char *cache_nonce,
   if (r_passphrase)
     *r_passphrase = NULL;
 
-  err = read_key_file (grip, &s_skey, &keymeta);
+  if (!grip && !ctrl->have_keygrip)
+    return gpg_error (GPG_ERR_NO_SECKEY);
+
+  err = read_key_file (grip? grip : ctrl->keygrip, &s_skey, &keymeta);
   if (err)
     {
       if (gpg_err_code (err) == GPG_ERR_ENOENT)
@@ -1004,6 +1007,39 @@ agent_key_from_file (ctrl_t ctrl, const char *cache_nonce,
       return err;
     }
 
+  if (ctrl->restricted && !grip)
+    {
+      char hexgrip[40+4+1];
+      char *prompt;
+      char *comment_buffer = NULL;
+      const char *comment = NULL;
+
+      bin2hex (ctrl->keygrip, 20, hexgrip);
+
+      if (keymeta && (comment = nvc_get_string (keymeta, "Label:")))
+        {
+          if (strchr (comment, '\n')
+              && (comment_buffer = linefeed_to_percent0A (comment)))
+            comment = comment_buffer;
+        }
+
+      prompt = xtryasprintf (L_("Remote process requested the use of key%%0A"
+                                "  %s%%0A"
+                                "  (%s)%%0A"
+                                "Do you want to allow this?"),
+                             hexgrip, comment? comment:"");
+
+      gcry_free (comment_buffer);
+
+      err = agent_get_confirmation (ctrl, prompt,
+                                    L_("Allow"), L_("Deny"), 0);
+      xfree (prompt);
+
+      if (err)
+        return err;
+    }
+
+
   /* For use with the protection functions we also need the key as an
      canonical encoded S-expression in a buffer.  Create this buffer
      now.  */
@@ -1078,8 +1114,9 @@ agent_key_from_file (ctrl_t ctrl, const char *cache_nonce,
 
 	if (!err)
 	  {
-	    err = unprotect (ctrl, cache_nonce, desc_text_final, &buf, grip,
-                            cache_mode, lookup_ttl, r_passphrase);
+            err = unprotect (ctrl, cache_nonce, desc_text_final, &buf,
+                             grip? grip : ctrl->keygrip,
+                             cache_mode, lookup_ttl, r_passphrase);
 	    if (err)
 	      log_error ("failed to unprotect the secret key: %s\n",
 			 gpg_strerror (err));
diff --git a/agent/pkdecrypt.c b/agent/pkdecrypt.c
index ec23daf83..adb08a956 100644
--- a/agent/pkdecrypt.c
+++ b/agent/pkdecrypt.c
@@ -67,7 +67,7 @@ agent_pkdecrypt (ctrl_t ctrl, const char *desc_text,
       log_printhex (ciphertext, ciphertextlen, "cipher: ");
     }
   rc = agent_key_from_file (ctrl, NULL, desc_text,
-                            ctrl->keygrip, &shadow_info,
+                            NULL, &shadow_info,
                             CACHE_MODE_NORMAL, NULL, &s_skey, NULL);
   if (rc)
     {
diff --git a/agent/pksign.c b/agent/pksign.c
index c94c1a197..7046f4545 100644
--- a/agent/pksign.c
+++ b/agent/pksign.c
@@ -314,7 +314,7 @@ agent_pksign_do (ctrl_t ctrl, const char *cache_nonce,
   if (!ctrl->have_keygrip)
     return gpg_error (GPG_ERR_NO_SECKEY);
 
-  err = agent_key_from_file (ctrl, cache_nonce, desc_text, ctrl->keygrip,
+  err = agent_key_from_file (ctrl, cache_nonce, desc_text, NULL,
                              &shadow_info, cache_mode, lookup_ttl,
                              &s_skey, NULL);
   if (gpg_err_code (err) == GPG_ERR_NO_SECKEY)