1 From 34e765977633b5f81845c0183af6d388d8225f00 Mon Sep 17 00:00:00 2001
 
   2 From: Cristian Stoica <cristian.stoica@freescale.com>
 
   3 Date: Mon, 23 Feb 2015 12:14:07 +0200
 
   4 Subject: [PATCH 14/15] remove redundant data copy for pkc operations
 
   6 This patch removes a copy of a pkc request that was
 
   7 allocated on the hot-path. The copy was not necessary
 
   8 and was just slowing things down.
 
  10 Change-Id: I3ad85f78c188f100ab9fc03a5777bb704a9dcb63
 
  11 Signed-off-by: Cristian Stoica <cristian.stoica@freescale.com>
 
  12 Reviewed-on: http://git.am.freescale.net:8181/34223
 
  14  cryptlib.c |  49 +++----------------
 
  17  main.c     | 162 +++++++++++++++++++++++++++++++++++++------------------------
 
  18  4 files changed, 113 insertions(+), 118 deletions(-)
 
  20 diff --git a/cryptlib.c b/cryptlib.c
 
  21 index 21e691b..5882a30 100644
 
  24 @@ -436,59 +436,22 @@ int cryptodev_hash_final(struct hash_data *hdata, void *output)
 
  26  int cryptodev_pkc_offload(struct cryptodev_pkc *pkc)
 
  29 -       struct pkc_request *pkc_req = &pkc->req, *pkc_requested;
 
  31 -       switch (pkc_req->type) {
 
  34 -       case RSA_PRIV_FORM1:
 
  35 -       case RSA_PRIV_FORM2:
 
  36 -       case RSA_PRIV_FORM3:
 
  37 -               pkc->s = crypto_alloc_pkc("pkc(rsa)", CRYPTO_ALG_TYPE_PKC_RSA, 0);
 
  45 -               pkc->s = crypto_alloc_pkc("pkc(dsa)", CRYPTO_ALG_TYPE_PKC_DSA, 0);
 
  47 -       case DH_COMPUTE_KEY:
 
  48 -       case ECDH_COMPUTE_KEY:
 
  49 -               pkc->s = crypto_alloc_pkc("pkc(dh)", CRYPTO_ALG_TYPE_PKC_DH, 0);
 
  55 -       if (IS_ERR_OR_NULL(pkc->s))
 
  59         init_completion(&pkc->result.completion);
 
  60 -       pkc_requested = pkc_request_alloc(pkc->s, GFP_KERNEL);
 
  62 -       if (unlikely(IS_ERR_OR_NULL(pkc_requested))) {
 
  66 -       pkc_requested->type = pkc_req->type;
 
  67 -       pkc_requested->curve_type = pkc_req->curve_type;
 
  68 -       memcpy(&pkc_requested->req_u, &pkc_req->req_u, sizeof(pkc_req->req_u));
 
  69 -       pkc_request_set_callback(pkc_requested, CRYPTO_TFM_REQ_MAY_BACKLOG,
 
  70 +       pkc_request_set_callback(pkc->req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 
  71                                  cryptodev_complete_asym, pkc);
 
  72 -       ret = crypto_pkc_op(pkc_requested);
 
  73 +       ret = crypto_pkc_op(pkc->req);
 
  74         if (ret != -EINPROGRESS && ret != 0)
 
  78         if (pkc->type == SYNCHRONOUS)
 
  79                 ret = waitfor(&pkc->result, ret);
 
  83 -       kfree(pkc_requested);
 
  87         crypto_free_pkc(pkc->s);
 
  90 diff --git a/cryptlib.h b/cryptlib.h
 
  91 index 7ffa54c..4fac0c8 100644
 
  94 @@ -110,8 +110,7 @@ struct cryptodev_pkc {
 
  95         struct crypto_pkc *s;    /* Transform pointer from CryptoAPI */
 
  96         struct cryptodev_result result; /* Result to be updated by
 
  98 -       struct pkc_request req; /* PKC request structure allocated
 
 100 +       struct pkc_request *req; /* PKC request allocated from CryptoAPI */
 
 101         enum offload_type type; /* Synchronous Vs Asynchronous request */
 
 103          * cookie used for transfering tranparent information from async
 
 104 diff --git a/ioctl.c b/ioctl.c
 
 105 index ee0486c..797b73c 100644
 
 108 @@ -708,26 +708,25 @@ static int crypto_async_fetch_asym(struct cryptodev_pkc *pkc)
 
 110         struct kernel_crypt_kop *kop = &pkc->kop;
 
 111         struct crypt_kop *ckop = &kop->kop;
 
 112 -       struct pkc_request *pkc_req = &pkc->req;
 
 114         switch (ckop->crk_op) {
 
 117 -               struct rsa_pub_req_s *rsa_req = &pkc_req->req_u.rsa_pub_req;
 
 118 +               struct rsa_pub_req_s *rsa_req = &pkc->req->req_u.rsa_pub_req;
 
 119                 copy_to_user(ckop->crk_param[3].crp_p, rsa_req->g, rsa_req->g_len);
 
 122         case CRK_MOD_EXP_CRT:
 
 124 -               struct rsa_priv_frm3_req_s *rsa_req = &pkc_req->req_u.rsa_priv_f3;
 
 125 +               struct rsa_priv_frm3_req_s *rsa_req = &pkc->req->req_u.rsa_priv_f3;
 
 126                 copy_to_user(ckop->crk_param[6].crp_p, rsa_req->f, rsa_req->f_len);
 
 131 -               struct dsa_sign_req_s *dsa_req = &pkc_req->req_u.dsa_sign;
 
 132 +               struct dsa_sign_req_s *dsa_req = &pkc->req->req_u.dsa_sign;
 
 134 -               if (pkc_req->type == ECDSA_SIGN) {
 
 135 +               if (pkc->req->type == ECDSA_SIGN) {
 
 136                         copy_to_user(ckop->crk_param[6].crp_p, dsa_req->c, dsa_req->d_len);
 
 137                         copy_to_user(ckop->crk_param[7].crp_p, dsa_req->d, dsa_req->d_len);
 
 139 @@ -740,8 +739,8 @@ static int crypto_async_fetch_asym(struct cryptodev_pkc *pkc)
 
 141         case CRK_DH_COMPUTE_KEY:
 
 143 -               struct dh_key_req_s *dh_req = &pkc_req->req_u.dh_req;
 
 144 -               if (pkc_req->type == ECDH_COMPUTE_KEY)
 
 145 +               struct dh_key_req_s *dh_req = &pkc->req->req_u.dh_req;
 
 146 +               if (pkc->req->type == ECDH_COMPUTE_KEY)
 
 147                         copy_to_user(ckop->crk_param[4].crp_p, dh_req->z, dh_req->z_len);
 
 149                         copy_to_user(ckop->crk_param[3].crp_p, dh_req->z, dh_req->z_len);
 
 150 @@ -750,9 +749,9 @@ static int crypto_async_fetch_asym(struct cryptodev_pkc *pkc)
 
 151         case CRK_DSA_GENERATE_KEY:
 
 152         case CRK_DH_GENERATE_KEY:
 
 154 -               struct keygen_req_s *key_req = &pkc_req->req_u.keygen;
 
 155 +               struct keygen_req_s *key_req = &pkc->req->req_u.keygen;
 
 157 -               if (pkc_req->type == ECC_KEYGEN) {
 
 158 +               if (pkc->req->type == ECC_KEYGEN) {
 
 159                         copy_to_user(ckop->crk_param[4].crp_p, key_req->pub_key,
 
 160                                         key_req->pub_key_len);
 
 161                         copy_to_user(ckop->crk_param[5].crp_p, key_req->priv_key,
 
 162 diff --git a/main.c b/main.c
 
 163 index af66553..ed1c69a 100644
 
 166 @@ -186,8 +186,7 @@ int crypto_kop_dsasign(struct cryptodev_pkc *pkc)
 
 168         struct kernel_crypt_kop *kop = &pkc->kop;
 
 169         struct crypt_kop *cop = &kop->kop;
 
 170 -       struct pkc_request *pkc_req = &pkc->req;
 
 171 -       struct dsa_sign_req_s *dsa_req = &pkc_req->req_u.dsa_sign;
 
 172 +       struct dsa_sign_req_s *dsa_req = &pkc->req->req_u.dsa_sign;
 
 176 @@ -210,10 +209,7 @@ int crypto_kop_dsasign(struct cryptodev_pkc *pkc)
 
 177         if (cop->crk_iparams == 6) {
 
 178                 dsa_req->ab_len = (cop->crk_param[5].crp_nbits + 7)/8;
 
 179                 buf_size += dsa_req->ab_len;
 
 180 -               pkc_req->type = ECDSA_SIGN;
 
 181 -               pkc_req->curve_type = cop->curve_type;
 
 183 -               pkc_req->type = DSA_SIGN;
 
 184 +               pkc->req->curve_type = cop->curve_type;
 
 187         buf = kmalloc(buf_size, GFP_DMA);
 
 188 @@ -269,7 +265,6 @@ int crypto_kop_dsaverify(struct cryptodev_pkc *pkc)
 
 190         struct kernel_crypt_kop *kop = &pkc->kop;
 
 191         struct crypt_kop *cop = &kop->kop;
 
 192 -       struct pkc_request *pkc_req;
 
 193         struct dsa_verify_req_s *dsa_req;
 
 196 @@ -281,8 +276,7 @@ int crypto_kop_dsaverify(struct cryptodev_pkc *pkc)
 
 197             !cop->crk_param[7].crp_nbits))
 
 200 -       pkc_req = &pkc->req;
 
 201 -       dsa_req = &pkc_req->req_u.dsa_verify;
 
 202 +       dsa_req = &pkc->req->req_u.dsa_verify;
 
 203         dsa_req->m_len = (cop->crk_param[0].crp_nbits + 7)/8;
 
 204         dsa_req->q_len = (cop->crk_param[1].crp_nbits + 7)/8;
 
 205         dsa_req->r_len = (cop->crk_param[2].crp_nbits + 7)/8;
 
 206 @@ -295,10 +289,7 @@ int crypto_kop_dsaverify(struct cryptodev_pkc *pkc)
 
 207         if (cop->crk_iparams == 8) {
 
 208                 dsa_req->ab_len = (cop->crk_param[5].crp_nbits + 7)/8;
 
 209                 buf_size += dsa_req->ab_len;
 
 210 -               pkc_req->type = ECDSA_VERIFY;
 
 211 -               pkc_req->curve_type = cop->curve_type;
 
 213 -               pkc_req->type = DSA_VERIFY;
 
 214 +               pkc->req->curve_type = cop->curve_type;
 
 217         buf = kmalloc(buf_size, GFP_DMA);
 
 218 @@ -351,7 +342,6 @@ int crypto_kop_rsa_keygen(struct cryptodev_pkc *pkc)
 
 220         struct kernel_crypt_kop *kop = &pkc->kop;
 
 221         struct crypt_kop *cop = &kop->kop;
 
 222 -       struct pkc_request *pkc_req;
 
 223         struct rsa_keygen_req_s *key_req;
 
 226 @@ -362,9 +352,7 @@ int crypto_kop_rsa_keygen(struct cryptodev_pkc *pkc)
 
 227                 !cop->crk_param[6].crp_nbits)
 
 230 -       pkc_req = &pkc->req;
 
 231 -       pkc_req->type = RSA_KEYGEN;
 
 232 -       key_req = &pkc_req->req_u.rsa_keygen;
 
 233 +       key_req = &pkc->req->req_u.rsa_keygen;
 
 234         key_req->n_len = (cop->crk_param[2].crp_nbits + 7)/8;
 
 235         key_req->p_len = (cop->crk_param[0].crp_nbits + 7) / 8;
 
 236         key_req->q_len = (cop->crk_param[1].crp_nbits + 7) / 8;
 
 237 @@ -427,7 +415,6 @@ int crypto_kop_keygen(struct cryptodev_pkc *pkc)
 
 239         struct kernel_crypt_kop *kop = &pkc->kop;
 
 240         struct crypt_kop *cop = &kop->kop;
 
 241 -       struct pkc_request *pkc_req;
 
 242         struct keygen_req_s *key_req;
 
 245 @@ -437,8 +424,7 @@ int crypto_kop_keygen(struct cryptodev_pkc *pkc)
 
 246             !cop->crk_param[4].crp_nbits)
 
 249 -       pkc_req = &pkc->req;
 
 250 -       key_req = &pkc_req->req_u.keygen;
 
 251 +       key_req = &pkc->req->req_u.keygen;
 
 252         key_req->q_len = (cop->crk_param[0].crp_nbits + 7)/8;
 
 253         key_req->r_len = (cop->crk_param[1].crp_nbits + 7)/8;
 
 254         key_req->g_len = (cop->crk_param[2].crp_nbits + 7)/8;
 
 255 @@ -447,7 +433,6 @@ int crypto_kop_keygen(struct cryptodev_pkc *pkc)
 
 256                 key_req->priv_key_len = (cop->crk_param[4].crp_nbits + 7)/8;
 
 257                 buf_size = key_req->q_len + key_req->r_len + key_req->g_len +
 
 258                         key_req->pub_key_len + key_req->priv_key_len;
 
 259 -               pkc_req->type = DLC_KEYGEN;
 
 261                 key_req->ab_len = (cop->crk_param[3].crp_nbits + 7)/8;
 
 262                 key_req->pub_key_len = (cop->crk_param[4].crp_nbits + 7)/8;
 
 263 @@ -455,8 +440,7 @@ int crypto_kop_keygen(struct cryptodev_pkc *pkc)
 
 264                 buf_size = key_req->q_len + key_req->r_len + key_req->g_len +
 
 265                         key_req->pub_key_len + key_req->priv_key_len +
 
 267 -               pkc_req->type = ECC_KEYGEN;
 
 268 -               pkc_req->curve_type = cop->curve_type;
 
 269 +               pkc->req->curve_type = cop->curve_type;
 
 272         buf = kmalloc(buf_size, GFP_DMA);
 
 273 @@ -508,26 +492,22 @@ int crypto_kop_dh_key(struct cryptodev_pkc *pkc)
 
 275         struct kernel_crypt_kop *kop = &pkc->kop;
 
 276         struct crypt_kop *cop = &kop->kop;
 
 277 -       struct pkc_request *pkc_req;
 
 278         struct dh_key_req_s *dh_req;
 
 283 -       pkc_req = &pkc->req;
 
 284 -       dh_req = &pkc_req->req_u.dh_req;
 
 285 +       dh_req = &pkc->req->req_u.dh_req;
 
 286         dh_req->s_len = (cop->crk_param[0].crp_nbits + 7)/8;
 
 287         dh_req->pub_key_len = (cop->crk_param[1].crp_nbits + 7)/8;
 
 288         dh_req->q_len = (cop->crk_param[2].crp_nbits + 7)/8;
 
 289         buf_size = dh_req->q_len + dh_req->pub_key_len + dh_req->s_len;
 
 290         if (cop->crk_iparams == 4) {
 
 291 -               pkc_req->type = ECDH_COMPUTE_KEY;
 
 292                 dh_req->ab_len = (cop->crk_param[3].crp_nbits + 7)/8;
 
 293                 dh_req->z_len = (cop->crk_param[4].crp_nbits + 7)/8;
 
 294                 buf_size += dh_req->ab_len;
 
 296                 dh_req->z_len = (cop->crk_param[3].crp_nbits + 7)/8;
 
 297 -               pkc_req->type = DH_COMPUTE_KEY;
 
 299         buf_size += dh_req->z_len;
 
 300         buf = kmalloc(buf_size, GFP_DMA);
 
 301 @@ -539,7 +519,7 @@ int crypto_kop_dh_key(struct cryptodev_pkc *pkc)
 
 302         dh_req->z = dh_req->pub_key + dh_req->pub_key_len;
 
 303         if (cop->crk_iparams == 4) {
 
 304                 dh_req->ab = dh_req->z + dh_req->z_len;
 
 305 -               pkc_req->curve_type = cop->curve_type;
 
 306 +               pkc->req->curve_type = cop->curve_type;
 
 307                 copy_from_user(dh_req->ab, cop->crk_param[3].crp_p,
 
 310 @@ -573,7 +553,6 @@ int crypto_modexp_crt(struct cryptodev_pkc *pkc)
 
 312         struct kernel_crypt_kop *kop = &pkc->kop;
 
 313         struct crypt_kop *cop = &kop->kop;
 
 314 -       struct pkc_request *pkc_req;
 
 315         struct rsa_priv_frm3_req_s *rsa_req;
 
 318 @@ -583,9 +562,7 @@ int crypto_modexp_crt(struct cryptodev_pkc *pkc)
 
 319             !cop->crk_param[4].crp_nbits || !cop->crk_param[5].crp_nbits)
 
 322 -       pkc_req = &pkc->req;
 
 323 -       pkc_req->type = RSA_PRIV_FORM3;
 
 324 -       rsa_req = &pkc_req->req_u.rsa_priv_f3;
 
 325 +       rsa_req = &pkc->req->req_u.rsa_priv_f3;
 
 326         rsa_req->p_len = (cop->crk_param[0].crp_nbits + 7)/8;
 
 327         rsa_req->q_len = (cop->crk_param[1].crp_nbits + 7)/8;
 
 328         rsa_req->g_len = (cop->crk_param[2].crp_nbits + 7)/8;
 
 329 @@ -632,7 +609,6 @@ err:
 
 331  int crypto_bn_modexp(struct cryptodev_pkc *pkc)
 
 333 -       struct pkc_request *pkc_req;
 
 334         struct rsa_pub_req_s *rsa_req;
 
 336         struct kernel_crypt_kop *kop = &pkc->kop;
 
 337 @@ -643,9 +619,7 @@ int crypto_bn_modexp(struct cryptodev_pkc *pkc)
 
 338             !cop->crk_param[2].crp_nbits || !cop->crk_param[3].crp_nbits)
 
 341 -       pkc_req = &pkc->req;
 
 342 -       pkc_req->type = RSA_PUB;
 
 343 -       rsa_req = &pkc_req->req_u.rsa_pub_req;
 
 344 +       rsa_req = &pkc->req->req_u.rsa_pub_req;
 
 345         rsa_req->f_len = (cop->crk_param[0].crp_nbits + 7)/8;
 
 346         rsa_req->e_len = (cop->crk_param[1].crp_nbits + 7)/8;
 
 347         rsa_req->n_len = (cop->crk_param[2].crp_nbits + 7)/8;
 
 348 @@ -680,56 +654,116 @@ err:
 
 356 +} pkc_alg_list[] = {
 
 357 +               {"pkc(rsa)", CRYPTO_ALG_TYPE_PKC_RSA, 0},
 
 358 +               {"pkc(dsa)", CRYPTO_ALG_TYPE_PKC_DSA, 0},
 
 359 +               {"pkc(dh)", CRYPTO_ALG_TYPE_PKC_DH, 0},
 
 362  int crypto_run_asym(struct cryptodev_pkc *pkc)
 
 367         struct kernel_crypt_kop *kop = &pkc->kop;
 
 368 +       enum pkc_req_type pkc_req_type;
 
 369 +       int (*call_next_action)(struct cryptodev_pkc *pkc);
 
 371         switch (kop->kop.crk_op) {
 
 373                 if (kop->kop.crk_iparams != 3 && kop->kop.crk_oparams != 1)
 
 376 -               ret = crypto_bn_modexp(pkc);
 
 378 +               pkc_req_type = RSA_PUB;
 
 380 +               call_next_action = crypto_bn_modexp;
 
 382         case CRK_MOD_EXP_CRT:
 
 383                 if (kop->kop.crk_iparams != 6 && kop->kop.crk_oparams != 1)
 
 386 -               ret = crypto_modexp_crt(pkc);
 
 388 +               pkc_req_type = RSA_PRIV_FORM3;
 
 390 +               call_next_action = crypto_modexp_crt;
 
 393 -               if ((kop->kop.crk_iparams != 5 && kop->kop.crk_iparams != 6) ||
 
 394 -                   kop->kop.crk_oparams != 2)
 
 397 -               ret = crypto_kop_dsasign(pkc);
 
 398 +               if (kop->kop.crk_oparams != 2)
 
 400 +               else if (kop->kop.crk_iparams == 5)
 
 401 +                       pkc_req_type = DSA_SIGN;
 
 402 +               else if (kop->kop.crk_iparams == 6)
 
 403 +                       pkc_req_type = ECDSA_SIGN;
 
 407 +               call_next_action = crypto_kop_dsasign;
 
 410 -               if ((kop->kop.crk_iparams != 7 && kop->kop.crk_iparams != 8) ||
 
 411 -                   kop->kop.crk_oparams != 0)
 
 414 -               ret = crypto_kop_dsaverify(pkc);
 
 415 +               if (kop->kop.crk_oparams != 0)
 
 417 +               else if (kop->kop.crk_iparams == 7)
 
 418 +                       pkc_req_type = DSA_VERIFY;
 
 419 +               else if (kop->kop.crk_iparams == 8)
 
 420 +                       pkc_req_type = ECDSA_VERIFY;
 
 424 +               call_next_action = crypto_kop_dsaverify;
 
 426         case CRK_DH_COMPUTE_KEY:
 
 427 -               if ((kop->kop.crk_iparams != 3 && kop->kop.crk_iparams != 4) ||
 
 428 -                   kop->kop.crk_oparams != 1)
 
 430 -               ret = crypto_kop_dh_key(pkc);
 
 431 +               if (kop->kop.crk_oparams != 1)
 
 433 +               else if (kop->kop.crk_iparams == 3)
 
 434 +                       pkc_req_type =  DH_COMPUTE_KEY;
 
 435 +               else if (kop->kop.crk_iparams == 4)
 
 436 +                       pkc_req_type =  ECDH_COMPUTE_KEY;
 
 440 +               call_next_action = crypto_kop_dh_key;
 
 442         case CRK_DH_GENERATE_KEY:
 
 443         case CRK_DSA_GENERATE_KEY:
 
 444 -               if ((kop->kop.crk_iparams != 3 && kop->kop.crk_iparams != 4))
 
 446 -               ret = crypto_kop_keygen(pkc);
 
 447 +               if (kop->kop.crk_iparams == 3)
 
 448 +                       pkc_req_type = DLC_KEYGEN;
 
 449 +               else if (kop->kop.crk_iparams == 4)
 
 450 +                       pkc_req_type = ECC_KEYGEN;
 
 454 +               call_next_action = crypto_kop_keygen;
 
 456         case CRK_RSA_GENERATE_KEY:
 
 457 -               ret = crypto_kop_rsa_keygen(pkc);
 
 458 +               pkc_req_type = RSA_KEYGEN;
 
 460 +               call_next_action = crypto_kop_rsa_keygen;
 
 468 +       pkc->s = crypto_alloc_pkc(pkc_alg_list[id].alg_name,
 
 469 +                                       pkc_alg_list[id].type,
 
 470 +                                       pkc_alg_list[id].mask);
 
 471 +       if (IS_ERR_OR_NULL(pkc->s))
 
 474 +       pkc->req = pkc_request_alloc(pkc->s, GFP_KERNEL);
 
 475 +       if (IS_ERR_OR_NULL(pkc->req))
 
 478 +       /* todo - fix alloc-free on error path */
 
 479 +       pkc->req->type = pkc_req_type;
 
 480 +       err = call_next_action(pkc);
 
 481 +       if (pkc->type == SYNCHRONOUS)
 
 487 +       crypto_free_pkc(pkc->s);
 
 491  int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop)