crypto: api - Keep failed instances alive

This patch reverts commit 9c521a20 ("crypto: api - remove
instance when test failed") and fixes the underlying problem
in a different way.

To recap, prior to the reverted commit, an instance that fails
a self-test is kept around.  However, it would satisfy any new
lookups against its name and therefore the system may accumlulate
an unbounded number of failed instances for the same algorithm
name.

The reverted commit fixed it by unregistering the instance.  Hoever,
this still does not prevent the creation of the same failed instance
over and over again each time the name is looked up.

This patch fixes it by keeping the failed instance around, just as
we would if it were a normal algorithm.  However, the lookup code
has been udpated so that we do not attempt to create another
instance as long as this failed one is still registered.  Of course,
you could still force a new creation by deleting the instance from
user-space.

A new error (ELIBBAD) has been commandeered for this purpose and
will be returned when all registered algorithm of a given name
have failed the self-test.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

crypto/algapi.c

@@ -543,9 +543,6 @@ int crypto_register_instance(struct crypto_template *tmpl,
 	inst->alg.cra_module = tmpl->module;
 	inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;
 
-	if (unlikely(!crypto_mod_get(&inst->alg)))
-		return -EAGAIN;
-
 	down_write(&crypto_alg_sem);
 
 	larval = __crypto_register_alg(&inst->alg);
@@ -563,14 +560,9 @@ int crypto_register_instance(struct crypto_template *tmpl,
 		goto err;
 
 	crypto_wait_for_test(larval);
-
-	/* Remove instance if test failed */
-	if (!(inst->alg.cra_flags & CRYPTO_ALG_TESTED))
-		crypto_unregister_instance(inst);
 	err = 0;
 
 err:
-	crypto_mod_put(&inst->alg);
 	return err;
 }
 EXPORT_SYMBOL_GPL(crypto_register_instance);

crypto/api.c

@@ -197,9 +197,16 @@ static struct crypto_alg *crypto_alg_lookup(const char *name, u32 type,
 					    u32 mask)
 {
 	struct crypto_alg *alg;
+	u32 test = 0;
+
+	if (!((type | mask) & CRYPTO_ALG_TESTED))
+		test |= CRYPTO_ALG_TESTED;
 
 	down_read(&crypto_alg_sem);
-	alg = __crypto_alg_lookup(name, type, mask);
+	alg = __crypto_alg_lookup(name, type | test, mask | test);
+	if (!alg && test)
+		alg = __crypto_alg_lookup(name, type, mask) ?
+		      ERR_PTR(-ELIBBAD) : NULL;
 	up_read(&crypto_alg_sem);
 
 	return alg;
@@ -227,10 +234,12 @@ static struct crypto_alg *crypto_larval_lookup(const char *name, u32 type,
 		alg = crypto_alg_lookup(name, type, mask);
 	}
 
-	if (alg)
-		return crypto_is_larval(alg) ? crypto_larval_wait(alg) : alg;
+	if (!IS_ERR_OR_NULL(alg) && crypto_is_larval(alg))
+		alg = crypto_larval_wait(alg);
+	else if (!alg)
+		alg = crypto_larval_add(name, type, mask);
 
-	return crypto_larval_add(name, type, mask);
+	return alg;
 }
 
 int crypto_probing_notify(unsigned long val, void *v)
@@ -253,11 +262,6 @@ struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask)
 	struct crypto_alg *larval;
 	int ok;
 
-	if (!((type | mask) & CRYPTO_ALG_TESTED)) {
-		type |= CRYPTO_ALG_TESTED;
-		mask |= CRYPTO_ALG_TESTED;
-	}
-
 	/*
 	 * If the internal flag is set for a cipher, require a caller to
 	 * to invoke the cipher with the internal flag to use that cipher.