1 From a4c561aab1014c3630bc88faf6f5246fee16b020 Mon Sep 17 00:00:00 2001
2 From: Jussi Kivilinna <jussi.kivilinna@iki.fi>
3 Date: Fri, 31 May 2019 17:27:25 +0300
4 Subject: [PATCH 3/3] GCM: move look-up table to .data section and unshare
7 * cipher/cipher-gcm.c (ATTR_ALIGNED_64): New.
8 (gcmR): Move to 'gcm_table' structure.
9 (gcm_table): New structure for look-up table with counters before and
12 (prefetch_table): Handle input with length not multiple of 256.
13 (do_prefetch_tables): Modify pre- and post-table counters to unshare
14 look-up table pages between processes.
18 Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>
20 Upstream-Status: Backport
21 [https://github.com/gpg/libgcrypt/commit/a4c561aab1014c3630bc88faf6f5246fee16b020]
25 Signed-off-by: Yi Zhao <yi.zhao@windriver.com>
27 cipher/cipher-gcm.c | 106 ++++++++++++++++++++++++++++++++++------------------
28 1 file changed, 70 insertions(+), 36 deletions(-)
30 diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c
31 index 11f119a..194e2ec 100644
32 --- a/cipher/cipher-gcm.c
33 +++ b/cipher/cipher-gcm.c
35 #include "./cipher-internal.h"
38 +/* Helper macro to force alignment to 16 or 64 bytes. */
39 +#ifdef HAVE_GCC_ATTRIBUTE_ALIGNED
40 +# define ATTR_ALIGNED_64 __attribute__ ((aligned (64)))
42 +# define ATTR_ALIGNED_64
46 #ifdef GCM_USE_INTEL_PCLMUL
47 extern void _gcry_ghash_setup_intel_pclmul (gcry_cipher_hd_t c);
49 @@ -83,40 +91,54 @@ ghash_armv7_neon (gcry_cipher_hd_t c, byte *result, const byte *buf,
53 -static const u16 gcmR[256] = {
54 - 0x0000, 0x01c2, 0x0384, 0x0246, 0x0708, 0x06ca, 0x048c, 0x054e,
55 - 0x0e10, 0x0fd2, 0x0d94, 0x0c56, 0x0918, 0x08da, 0x0a9c, 0x0b5e,
56 - 0x1c20, 0x1de2, 0x1fa4, 0x1e66, 0x1b28, 0x1aea, 0x18ac, 0x196e,
57 - 0x1230, 0x13f2, 0x11b4, 0x1076, 0x1538, 0x14fa, 0x16bc, 0x177e,
58 - 0x3840, 0x3982, 0x3bc4, 0x3a06, 0x3f48, 0x3e8a, 0x3ccc, 0x3d0e,
59 - 0x3650, 0x3792, 0x35d4, 0x3416, 0x3158, 0x309a, 0x32dc, 0x331e,
60 - 0x2460, 0x25a2, 0x27e4, 0x2626, 0x2368, 0x22aa, 0x20ec, 0x212e,
61 - 0x2a70, 0x2bb2, 0x29f4, 0x2836, 0x2d78, 0x2cba, 0x2efc, 0x2f3e,
62 - 0x7080, 0x7142, 0x7304, 0x72c6, 0x7788, 0x764a, 0x740c, 0x75ce,
63 - 0x7e90, 0x7f52, 0x7d14, 0x7cd6, 0x7998, 0x785a, 0x7a1c, 0x7bde,
64 - 0x6ca0, 0x6d62, 0x6f24, 0x6ee6, 0x6ba8, 0x6a6a, 0x682c, 0x69ee,
65 - 0x62b0, 0x6372, 0x6134, 0x60f6, 0x65b8, 0x647a, 0x663c, 0x67fe,
66 - 0x48c0, 0x4902, 0x4b44, 0x4a86, 0x4fc8, 0x4e0a, 0x4c4c, 0x4d8e,
67 - 0x46d0, 0x4712, 0x4554, 0x4496, 0x41d8, 0x401a, 0x425c, 0x439e,
68 - 0x54e0, 0x5522, 0x5764, 0x56a6, 0x53e8, 0x522a, 0x506c, 0x51ae,
69 - 0x5af0, 0x5b32, 0x5974, 0x58b6, 0x5df8, 0x5c3a, 0x5e7c, 0x5fbe,
70 - 0xe100, 0xe0c2, 0xe284, 0xe346, 0xe608, 0xe7ca, 0xe58c, 0xe44e,
71 - 0xef10, 0xeed2, 0xec94, 0xed56, 0xe818, 0xe9da, 0xeb9c, 0xea5e,
72 - 0xfd20, 0xfce2, 0xfea4, 0xff66, 0xfa28, 0xfbea, 0xf9ac, 0xf86e,
73 - 0xf330, 0xf2f2, 0xf0b4, 0xf176, 0xf438, 0xf5fa, 0xf7bc, 0xf67e,
74 - 0xd940, 0xd882, 0xdac4, 0xdb06, 0xde48, 0xdf8a, 0xddcc, 0xdc0e,
75 - 0xd750, 0xd692, 0xd4d4, 0xd516, 0xd058, 0xd19a, 0xd3dc, 0xd21e,
76 - 0xc560, 0xc4a2, 0xc6e4, 0xc726, 0xc268, 0xc3aa, 0xc1ec, 0xc02e,
77 - 0xcb70, 0xcab2, 0xc8f4, 0xc936, 0xcc78, 0xcdba, 0xcffc, 0xce3e,
78 - 0x9180, 0x9042, 0x9204, 0x93c6, 0x9688, 0x974a, 0x950c, 0x94ce,
79 - 0x9f90, 0x9e52, 0x9c14, 0x9dd6, 0x9898, 0x995a, 0x9b1c, 0x9ade,
80 - 0x8da0, 0x8c62, 0x8e24, 0x8fe6, 0x8aa8, 0x8b6a, 0x892c, 0x88ee,
81 - 0x83b0, 0x8272, 0x8034, 0x81f6, 0x84b8, 0x857a, 0x873c, 0x86fe,
82 - 0xa9c0, 0xa802, 0xaa44, 0xab86, 0xaec8, 0xaf0a, 0xad4c, 0xac8e,
83 - 0xa7d0, 0xa612, 0xa454, 0xa596, 0xa0d8, 0xa11a, 0xa35c, 0xa29e,
84 - 0xb5e0, 0xb422, 0xb664, 0xb7a6, 0xb2e8, 0xb32a, 0xb16c, 0xb0ae,
85 - 0xbbf0, 0xba32, 0xb874, 0xb9b6, 0xbcf8, 0xbd3a, 0xbf7c, 0xbebe,
89 + volatile u32 counter_head;
90 + u32 cacheline_align[64 / 4 - 1];
92 + volatile u32 counter_tail;
93 +} gcm_table ATTR_ALIGNED_64 =
98 + 0x0000, 0x01c2, 0x0384, 0x0246, 0x0708, 0x06ca, 0x048c, 0x054e,
99 + 0x0e10, 0x0fd2, 0x0d94, 0x0c56, 0x0918, 0x08da, 0x0a9c, 0x0b5e,
100 + 0x1c20, 0x1de2, 0x1fa4, 0x1e66, 0x1b28, 0x1aea, 0x18ac, 0x196e,
101 + 0x1230, 0x13f2, 0x11b4, 0x1076, 0x1538, 0x14fa, 0x16bc, 0x177e,
102 + 0x3840, 0x3982, 0x3bc4, 0x3a06, 0x3f48, 0x3e8a, 0x3ccc, 0x3d0e,
103 + 0x3650, 0x3792, 0x35d4, 0x3416, 0x3158, 0x309a, 0x32dc, 0x331e,
104 + 0x2460, 0x25a2, 0x27e4, 0x2626, 0x2368, 0x22aa, 0x20ec, 0x212e,
105 + 0x2a70, 0x2bb2, 0x29f4, 0x2836, 0x2d78, 0x2cba, 0x2efc, 0x2f3e,
106 + 0x7080, 0x7142, 0x7304, 0x72c6, 0x7788, 0x764a, 0x740c, 0x75ce,
107 + 0x7e90, 0x7f52, 0x7d14, 0x7cd6, 0x7998, 0x785a, 0x7a1c, 0x7bde,
108 + 0x6ca0, 0x6d62, 0x6f24, 0x6ee6, 0x6ba8, 0x6a6a, 0x682c, 0x69ee,
109 + 0x62b0, 0x6372, 0x6134, 0x60f6, 0x65b8, 0x647a, 0x663c, 0x67fe,
110 + 0x48c0, 0x4902, 0x4b44, 0x4a86, 0x4fc8, 0x4e0a, 0x4c4c, 0x4d8e,
111 + 0x46d0, 0x4712, 0x4554, 0x4496, 0x41d8, 0x401a, 0x425c, 0x439e,
112 + 0x54e0, 0x5522, 0x5764, 0x56a6, 0x53e8, 0x522a, 0x506c, 0x51ae,
113 + 0x5af0, 0x5b32, 0x5974, 0x58b6, 0x5df8, 0x5c3a, 0x5e7c, 0x5fbe,
114 + 0xe100, 0xe0c2, 0xe284, 0xe346, 0xe608, 0xe7ca, 0xe58c, 0xe44e,
115 + 0xef10, 0xeed2, 0xec94, 0xed56, 0xe818, 0xe9da, 0xeb9c, 0xea5e,
116 + 0xfd20, 0xfce2, 0xfea4, 0xff66, 0xfa28, 0xfbea, 0xf9ac, 0xf86e,
117 + 0xf330, 0xf2f2, 0xf0b4, 0xf176, 0xf438, 0xf5fa, 0xf7bc, 0xf67e,
118 + 0xd940, 0xd882, 0xdac4, 0xdb06, 0xde48, 0xdf8a, 0xddcc, 0xdc0e,
119 + 0xd750, 0xd692, 0xd4d4, 0xd516, 0xd058, 0xd19a, 0xd3dc, 0xd21e,
120 + 0xc560, 0xc4a2, 0xc6e4, 0xc726, 0xc268, 0xc3aa, 0xc1ec, 0xc02e,
121 + 0xcb70, 0xcab2, 0xc8f4, 0xc936, 0xcc78, 0xcdba, 0xcffc, 0xce3e,
122 + 0x9180, 0x9042, 0x9204, 0x93c6, 0x9688, 0x974a, 0x950c, 0x94ce,
123 + 0x9f90, 0x9e52, 0x9c14, 0x9dd6, 0x9898, 0x995a, 0x9b1c, 0x9ade,
124 + 0x8da0, 0x8c62, 0x8e24, 0x8fe6, 0x8aa8, 0x8b6a, 0x892c, 0x88ee,
125 + 0x83b0, 0x8272, 0x8034, 0x81f6, 0x84b8, 0x857a, 0x873c, 0x86fe,
126 + 0xa9c0, 0xa802, 0xaa44, 0xab86, 0xaec8, 0xaf0a, 0xad4c, 0xac8e,
127 + 0xa7d0, 0xa612, 0xa454, 0xa596, 0xa0d8, 0xa11a, 0xa35c, 0xa29e,
128 + 0xb5e0, 0xb422, 0xb664, 0xb7a6, 0xb2e8, 0xb32a, 0xb16c, 0xb0ae,
129 + 0xbbf0, 0xba32, 0xb874, 0xb9b6, 0xbcf8, 0xbd3a, 0xbf7c, 0xbebe,
134 +#define gcmR gcm_table.R
137 void prefetch_table(const void *tab, size_t len)
138 @@ -124,7 +146,7 @@ void prefetch_table(const void *tab, size_t len)
139 const volatile byte *vtab = tab;
142 - for (i = 0; i < len; i += 8 * 32)
143 + for (i = 0; len - i >= 8 * 32; i += 8 * 32)
145 (void)vtab[i + 0 * 32];
146 (void)vtab[i + 1 * 32];
147 @@ -135,6 +157,10 @@ void prefetch_table(const void *tab, size_t len)
148 (void)vtab[i + 6 * 32];
149 (void)vtab[i + 7 * 32];
151 + for (; i < len; i += 32)
158 @@ -142,8 +168,16 @@ void prefetch_table(const void *tab, size_t len)
160 do_prefetch_tables (const void *gcmM, size_t gcmM_size)
162 + /* Modify counters to trigger copy-on-write and unsharing if physical pages
163 + * of look-up table are shared between processes. Modifying counters also
164 + * causes checksums for pages to change and hint same-page merging algorithm
165 + * that these pages are frequently changing. */
166 + gcm_table.counter_head++;
167 + gcm_table.counter_tail++;
169 + /* Prefetch look-up tables to cache. */
170 prefetch_table(gcmM, gcmM_size);
171 - prefetch_table(gcmR, sizeof(gcmR));
172 + prefetch_table(&gcm_table, sizeof(gcm_table));
175 #ifdef GCM_TABLES_USE_U64
