1 From 34e765977633b5f81845c0183af6d388d8225f00 Mon Sep 17 00:00:00 2001
2 From: Cristian Stoica <cristian.stoica@freescale.com>
3 Date: Mon, 23 Feb 2015 12:14:07 +0200
4 Subject: [PATCH 14/15] remove redundant data copy for pkc operations
6 This patch removes a copy of a pkc request that was
7 allocated on the hot-path. The copy was not necessary
8 and was just slowing things down.
10 Change-Id: I3ad85f78c188f100ab9fc03a5777bb704a9dcb63
11 Signed-off-by: Cristian Stoica <cristian.stoica@freescale.com>
12 Reviewed-on: http://git.am.freescale.net:8181/34223
14 cryptlib.c | 49 +++----------------
17 main.c | 162 +++++++++++++++++++++++++++++++++++++------------------------
18 4 files changed, 113 insertions(+), 118 deletions(-)
20 diff --git a/cryptlib.c b/cryptlib.c
21 index 21e691b..5882a30 100644
24 @@ -436,59 +436,22 @@ int cryptodev_hash_final(struct hash_data *hdata, void *output)
26 int cryptodev_pkc_offload(struct cryptodev_pkc *pkc)
29 - struct pkc_request *pkc_req = &pkc->req, *pkc_requested;
31 - switch (pkc_req->type) {
34 - case RSA_PRIV_FORM1:
35 - case RSA_PRIV_FORM2:
36 - case RSA_PRIV_FORM3:
37 - pkc->s = crypto_alloc_pkc("pkc(rsa)", CRYPTO_ALG_TYPE_PKC_RSA, 0);
45 - pkc->s = crypto_alloc_pkc("pkc(dsa)", CRYPTO_ALG_TYPE_PKC_DSA, 0);
47 - case DH_COMPUTE_KEY:
48 - case ECDH_COMPUTE_KEY:
49 - pkc->s = crypto_alloc_pkc("pkc(dh)", CRYPTO_ALG_TYPE_PKC_DH, 0);
55 - if (IS_ERR_OR_NULL(pkc->s))
59 init_completion(&pkc->result.completion);
60 - pkc_requested = pkc_request_alloc(pkc->s, GFP_KERNEL);
62 - if (unlikely(IS_ERR_OR_NULL(pkc_requested))) {
66 - pkc_requested->type = pkc_req->type;
67 - pkc_requested->curve_type = pkc_req->curve_type;
68 - memcpy(&pkc_requested->req_u, &pkc_req->req_u, sizeof(pkc_req->req_u));
69 - pkc_request_set_callback(pkc_requested, CRYPTO_TFM_REQ_MAY_BACKLOG,
70 + pkc_request_set_callback(pkc->req, CRYPTO_TFM_REQ_MAY_BACKLOG,
71 cryptodev_complete_asym, pkc);
72 - ret = crypto_pkc_op(pkc_requested);
73 + ret = crypto_pkc_op(pkc->req);
74 if (ret != -EINPROGRESS && ret != 0)
78 if (pkc->type == SYNCHRONOUS)
79 ret = waitfor(&pkc->result, ret);
83 - kfree(pkc_requested);
87 crypto_free_pkc(pkc->s);
90 diff --git a/cryptlib.h b/cryptlib.h
91 index 7ffa54c..4fac0c8 100644
94 @@ -110,8 +110,7 @@ struct cryptodev_pkc {
95 struct crypto_pkc *s; /* Transform pointer from CryptoAPI */
96 struct cryptodev_result result; /* Result to be updated by
98 - struct pkc_request req; /* PKC request structure allocated
100 + struct pkc_request *req; /* PKC request allocated from CryptoAPI */
101 enum offload_type type; /* Synchronous Vs Asynchronous request */
103 * cookie used for transfering tranparent information from async
104 diff --git a/ioctl.c b/ioctl.c
105 index ee0486c..797b73c 100644
108 @@ -708,26 +708,25 @@ static int crypto_async_fetch_asym(struct cryptodev_pkc *pkc)
110 struct kernel_crypt_kop *kop = &pkc->kop;
111 struct crypt_kop *ckop = &kop->kop;
112 - struct pkc_request *pkc_req = &pkc->req;
114 switch (ckop->crk_op) {
117 - struct rsa_pub_req_s *rsa_req = &pkc_req->req_u.rsa_pub_req;
118 + struct rsa_pub_req_s *rsa_req = &pkc->req->req_u.rsa_pub_req;
119 copy_to_user(ckop->crk_param[3].crp_p, rsa_req->g, rsa_req->g_len);
122 case CRK_MOD_EXP_CRT:
124 - struct rsa_priv_frm3_req_s *rsa_req = &pkc_req->req_u.rsa_priv_f3;
125 + struct rsa_priv_frm3_req_s *rsa_req = &pkc->req->req_u.rsa_priv_f3;
126 copy_to_user(ckop->crk_param[6].crp_p, rsa_req->f, rsa_req->f_len);
131 - struct dsa_sign_req_s *dsa_req = &pkc_req->req_u.dsa_sign;
132 + struct dsa_sign_req_s *dsa_req = &pkc->req->req_u.dsa_sign;
134 - if (pkc_req->type == ECDSA_SIGN) {
135 + if (pkc->req->type == ECDSA_SIGN) {
136 copy_to_user(ckop->crk_param[6].crp_p, dsa_req->c, dsa_req->d_len);
137 copy_to_user(ckop->crk_param[7].crp_p, dsa_req->d, dsa_req->d_len);
139 @@ -740,8 +739,8 @@ static int crypto_async_fetch_asym(struct cryptodev_pkc *pkc)
141 case CRK_DH_COMPUTE_KEY:
143 - struct dh_key_req_s *dh_req = &pkc_req->req_u.dh_req;
144 - if (pkc_req->type == ECDH_COMPUTE_KEY)
145 + struct dh_key_req_s *dh_req = &pkc->req->req_u.dh_req;
146 + if (pkc->req->type == ECDH_COMPUTE_KEY)
147 copy_to_user(ckop->crk_param[4].crp_p, dh_req->z, dh_req->z_len);
149 copy_to_user(ckop->crk_param[3].crp_p, dh_req->z, dh_req->z_len);
150 @@ -750,9 +749,9 @@ static int crypto_async_fetch_asym(struct cryptodev_pkc *pkc)
151 case CRK_DSA_GENERATE_KEY:
152 case CRK_DH_GENERATE_KEY:
154 - struct keygen_req_s *key_req = &pkc_req->req_u.keygen;
155 + struct keygen_req_s *key_req = &pkc->req->req_u.keygen;
157 - if (pkc_req->type == ECC_KEYGEN) {
158 + if (pkc->req->type == ECC_KEYGEN) {
159 copy_to_user(ckop->crk_param[4].crp_p, key_req->pub_key,
160 key_req->pub_key_len);
161 copy_to_user(ckop->crk_param[5].crp_p, key_req->priv_key,
162 diff --git a/main.c b/main.c
163 index af66553..ed1c69a 100644
166 @@ -186,8 +186,7 @@ int crypto_kop_dsasign(struct cryptodev_pkc *pkc)
168 struct kernel_crypt_kop *kop = &pkc->kop;
169 struct crypt_kop *cop = &kop->kop;
170 - struct pkc_request *pkc_req = &pkc->req;
171 - struct dsa_sign_req_s *dsa_req = &pkc_req->req_u.dsa_sign;
172 + struct dsa_sign_req_s *dsa_req = &pkc->req->req_u.dsa_sign;
176 @@ -210,10 +209,7 @@ int crypto_kop_dsasign(struct cryptodev_pkc *pkc)
177 if (cop->crk_iparams == 6) {
178 dsa_req->ab_len = (cop->crk_param[5].crp_nbits + 7)/8;
179 buf_size += dsa_req->ab_len;
180 - pkc_req->type = ECDSA_SIGN;
181 - pkc_req->curve_type = cop->curve_type;
183 - pkc_req->type = DSA_SIGN;
184 + pkc->req->curve_type = cop->curve_type;
187 buf = kmalloc(buf_size, GFP_DMA);
188 @@ -269,7 +265,6 @@ int crypto_kop_dsaverify(struct cryptodev_pkc *pkc)
190 struct kernel_crypt_kop *kop = &pkc->kop;
191 struct crypt_kop *cop = &kop->kop;
192 - struct pkc_request *pkc_req;
193 struct dsa_verify_req_s *dsa_req;
196 @@ -281,8 +276,7 @@ int crypto_kop_dsaverify(struct cryptodev_pkc *pkc)
197 !cop->crk_param[7].crp_nbits))
200 - pkc_req = &pkc->req;
201 - dsa_req = &pkc_req->req_u.dsa_verify;
202 + dsa_req = &pkc->req->req_u.dsa_verify;
203 dsa_req->m_len = (cop->crk_param[0].crp_nbits + 7)/8;
204 dsa_req->q_len = (cop->crk_param[1].crp_nbits + 7)/8;
205 dsa_req->r_len = (cop->crk_param[2].crp_nbits + 7)/8;
206 @@ -295,10 +289,7 @@ int crypto_kop_dsaverify(struct cryptodev_pkc *pkc)
207 if (cop->crk_iparams == 8) {
208 dsa_req->ab_len = (cop->crk_param[5].crp_nbits + 7)/8;
209 buf_size += dsa_req->ab_len;
210 - pkc_req->type = ECDSA_VERIFY;
211 - pkc_req->curve_type = cop->curve_type;
213 - pkc_req->type = DSA_VERIFY;
214 + pkc->req->curve_type = cop->curve_type;
217 buf = kmalloc(buf_size, GFP_DMA);
218 @@ -351,7 +342,6 @@ int crypto_kop_rsa_keygen(struct cryptodev_pkc *pkc)
220 struct kernel_crypt_kop *kop = &pkc->kop;
221 struct crypt_kop *cop = &kop->kop;
222 - struct pkc_request *pkc_req;
223 struct rsa_keygen_req_s *key_req;
226 @@ -362,9 +352,7 @@ int crypto_kop_rsa_keygen(struct cryptodev_pkc *pkc)
227 !cop->crk_param[6].crp_nbits)
230 - pkc_req = &pkc->req;
231 - pkc_req->type = RSA_KEYGEN;
232 - key_req = &pkc_req->req_u.rsa_keygen;
233 + key_req = &pkc->req->req_u.rsa_keygen;
234 key_req->n_len = (cop->crk_param[2].crp_nbits + 7)/8;
235 key_req->p_len = (cop->crk_param[0].crp_nbits + 7) / 8;
236 key_req->q_len = (cop->crk_param[1].crp_nbits + 7) / 8;
237 @@ -427,7 +415,6 @@ int crypto_kop_keygen(struct cryptodev_pkc *pkc)
239 struct kernel_crypt_kop *kop = &pkc->kop;
240 struct crypt_kop *cop = &kop->kop;
241 - struct pkc_request *pkc_req;
242 struct keygen_req_s *key_req;
245 @@ -437,8 +424,7 @@ int crypto_kop_keygen(struct cryptodev_pkc *pkc)
246 !cop->crk_param[4].crp_nbits)
249 - pkc_req = &pkc->req;
250 - key_req = &pkc_req->req_u.keygen;
251 + key_req = &pkc->req->req_u.keygen;
252 key_req->q_len = (cop->crk_param[0].crp_nbits + 7)/8;
253 key_req->r_len = (cop->crk_param[1].crp_nbits + 7)/8;
254 key_req->g_len = (cop->crk_param[2].crp_nbits + 7)/8;
255 @@ -447,7 +433,6 @@ int crypto_kop_keygen(struct cryptodev_pkc *pkc)
256 key_req->priv_key_len = (cop->crk_param[4].crp_nbits + 7)/8;
257 buf_size = key_req->q_len + key_req->r_len + key_req->g_len +
258 key_req->pub_key_len + key_req->priv_key_len;
259 - pkc_req->type = DLC_KEYGEN;
261 key_req->ab_len = (cop->crk_param[3].crp_nbits + 7)/8;
262 key_req->pub_key_len = (cop->crk_param[4].crp_nbits + 7)/8;
263 @@ -455,8 +440,7 @@ int crypto_kop_keygen(struct cryptodev_pkc *pkc)
264 buf_size = key_req->q_len + key_req->r_len + key_req->g_len +
265 key_req->pub_key_len + key_req->priv_key_len +
267 - pkc_req->type = ECC_KEYGEN;
268 - pkc_req->curve_type = cop->curve_type;
269 + pkc->req->curve_type = cop->curve_type;
272 buf = kmalloc(buf_size, GFP_DMA);
273 @@ -508,26 +492,22 @@ int crypto_kop_dh_key(struct cryptodev_pkc *pkc)
275 struct kernel_crypt_kop *kop = &pkc->kop;
276 struct crypt_kop *cop = &kop->kop;
277 - struct pkc_request *pkc_req;
278 struct dh_key_req_s *dh_req;
283 - pkc_req = &pkc->req;
284 - dh_req = &pkc_req->req_u.dh_req;
285 + dh_req = &pkc->req->req_u.dh_req;
286 dh_req->s_len = (cop->crk_param[0].crp_nbits + 7)/8;
287 dh_req->pub_key_len = (cop->crk_param[1].crp_nbits + 7)/8;
288 dh_req->q_len = (cop->crk_param[2].crp_nbits + 7)/8;
289 buf_size = dh_req->q_len + dh_req->pub_key_len + dh_req->s_len;
290 if (cop->crk_iparams == 4) {
291 - pkc_req->type = ECDH_COMPUTE_KEY;
292 dh_req->ab_len = (cop->crk_param[3].crp_nbits + 7)/8;
293 dh_req->z_len = (cop->crk_param[4].crp_nbits + 7)/8;
294 buf_size += dh_req->ab_len;
296 dh_req->z_len = (cop->crk_param[3].crp_nbits + 7)/8;
297 - pkc_req->type = DH_COMPUTE_KEY;
299 buf_size += dh_req->z_len;
300 buf = kmalloc(buf_size, GFP_DMA);
301 @@ -539,7 +519,7 @@ int crypto_kop_dh_key(struct cryptodev_pkc *pkc)
302 dh_req->z = dh_req->pub_key + dh_req->pub_key_len;
303 if (cop->crk_iparams == 4) {
304 dh_req->ab = dh_req->z + dh_req->z_len;
305 - pkc_req->curve_type = cop->curve_type;
306 + pkc->req->curve_type = cop->curve_type;
307 copy_from_user(dh_req->ab, cop->crk_param[3].crp_p,
310 @@ -573,7 +553,6 @@ int crypto_modexp_crt(struct cryptodev_pkc *pkc)
312 struct kernel_crypt_kop *kop = &pkc->kop;
313 struct crypt_kop *cop = &kop->kop;
314 - struct pkc_request *pkc_req;
315 struct rsa_priv_frm3_req_s *rsa_req;
318 @@ -583,9 +562,7 @@ int crypto_modexp_crt(struct cryptodev_pkc *pkc)
319 !cop->crk_param[4].crp_nbits || !cop->crk_param[5].crp_nbits)
322 - pkc_req = &pkc->req;
323 - pkc_req->type = RSA_PRIV_FORM3;
324 - rsa_req = &pkc_req->req_u.rsa_priv_f3;
325 + rsa_req = &pkc->req->req_u.rsa_priv_f3;
326 rsa_req->p_len = (cop->crk_param[0].crp_nbits + 7)/8;
327 rsa_req->q_len = (cop->crk_param[1].crp_nbits + 7)/8;
328 rsa_req->g_len = (cop->crk_param[2].crp_nbits + 7)/8;
329 @@ -632,7 +609,6 @@ err:
331 int crypto_bn_modexp(struct cryptodev_pkc *pkc)
333 - struct pkc_request *pkc_req;
334 struct rsa_pub_req_s *rsa_req;
336 struct kernel_crypt_kop *kop = &pkc->kop;
337 @@ -643,9 +619,7 @@ int crypto_bn_modexp(struct cryptodev_pkc *pkc)
338 !cop->crk_param[2].crp_nbits || !cop->crk_param[3].crp_nbits)
341 - pkc_req = &pkc->req;
342 - pkc_req->type = RSA_PUB;
343 - rsa_req = &pkc_req->req_u.rsa_pub_req;
344 + rsa_req = &pkc->req->req_u.rsa_pub_req;
345 rsa_req->f_len = (cop->crk_param[0].crp_nbits + 7)/8;
346 rsa_req->e_len = (cop->crk_param[1].crp_nbits + 7)/8;
347 rsa_req->n_len = (cop->crk_param[2].crp_nbits + 7)/8;
348 @@ -680,56 +654,116 @@ err:
356 +} pkc_alg_list[] = {
357 + {"pkc(rsa)", CRYPTO_ALG_TYPE_PKC_RSA, 0},
358 + {"pkc(dsa)", CRYPTO_ALG_TYPE_PKC_DSA, 0},
359 + {"pkc(dh)", CRYPTO_ALG_TYPE_PKC_DH, 0},
362 int crypto_run_asym(struct cryptodev_pkc *pkc)
367 struct kernel_crypt_kop *kop = &pkc->kop;
368 + enum pkc_req_type pkc_req_type;
369 + int (*call_next_action)(struct cryptodev_pkc *pkc);
371 switch (kop->kop.crk_op) {
373 if (kop->kop.crk_iparams != 3 && kop->kop.crk_oparams != 1)
376 - ret = crypto_bn_modexp(pkc);
378 + pkc_req_type = RSA_PUB;
380 + call_next_action = crypto_bn_modexp;
382 case CRK_MOD_EXP_CRT:
383 if (kop->kop.crk_iparams != 6 && kop->kop.crk_oparams != 1)
386 - ret = crypto_modexp_crt(pkc);
388 + pkc_req_type = RSA_PRIV_FORM3;
390 + call_next_action = crypto_modexp_crt;
393 - if ((kop->kop.crk_iparams != 5 && kop->kop.crk_iparams != 6) ||
394 - kop->kop.crk_oparams != 2)
397 - ret = crypto_kop_dsasign(pkc);
398 + if (kop->kop.crk_oparams != 2)
400 + else if (kop->kop.crk_iparams == 5)
401 + pkc_req_type = DSA_SIGN;
402 + else if (kop->kop.crk_iparams == 6)
403 + pkc_req_type = ECDSA_SIGN;
407 + call_next_action = crypto_kop_dsasign;
410 - if ((kop->kop.crk_iparams != 7 && kop->kop.crk_iparams != 8) ||
411 - kop->kop.crk_oparams != 0)
414 - ret = crypto_kop_dsaverify(pkc);
415 + if (kop->kop.crk_oparams != 0)
417 + else if (kop->kop.crk_iparams == 7)
418 + pkc_req_type = DSA_VERIFY;
419 + else if (kop->kop.crk_iparams == 8)
420 + pkc_req_type = ECDSA_VERIFY;
424 + call_next_action = crypto_kop_dsaverify;
426 case CRK_DH_COMPUTE_KEY:
427 - if ((kop->kop.crk_iparams != 3 && kop->kop.crk_iparams != 4) ||
428 - kop->kop.crk_oparams != 1)
430 - ret = crypto_kop_dh_key(pkc);
431 + if (kop->kop.crk_oparams != 1)
433 + else if (kop->kop.crk_iparams == 3)
434 + pkc_req_type = DH_COMPUTE_KEY;
435 + else if (kop->kop.crk_iparams == 4)
436 + pkc_req_type = ECDH_COMPUTE_KEY;
440 + call_next_action = crypto_kop_dh_key;
442 case CRK_DH_GENERATE_KEY:
443 case CRK_DSA_GENERATE_KEY:
444 - if ((kop->kop.crk_iparams != 3 && kop->kop.crk_iparams != 4))
446 - ret = crypto_kop_keygen(pkc);
447 + if (kop->kop.crk_iparams == 3)
448 + pkc_req_type = DLC_KEYGEN;
449 + else if (kop->kop.crk_iparams == 4)
450 + pkc_req_type = ECC_KEYGEN;
454 + call_next_action = crypto_kop_keygen;
456 case CRK_RSA_GENERATE_KEY:
457 - ret = crypto_kop_rsa_keygen(pkc);
458 + pkc_req_type = RSA_KEYGEN;
460 + call_next_action = crypto_kop_rsa_keygen;
468 + pkc->s = crypto_alloc_pkc(pkc_alg_list[id].alg_name,
469 + pkc_alg_list[id].type,
470 + pkc_alg_list[id].mask);
471 + if (IS_ERR_OR_NULL(pkc->s))
474 + pkc->req = pkc_request_alloc(pkc->s, GFP_KERNEL);
475 + if (IS_ERR_OR_NULL(pkc->req))
478 + /* todo - fix alloc-free on error path */
479 + pkc->req->type = pkc_req_type;
480 + err = call_next_action(pkc);
481 + if (pkc->type == SYNCHRONOUS)
487 + crypto_free_pkc(pkc->s);
491 int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop)