-
Notifications
You must be signed in to change notification settings - Fork 155
/
rsa.c
211 lines (177 loc) · 8 KB
/
rsa.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
#include <string.h>
#include <stdlib.h>
#include "rsa.h"
#include "sha.h"
#include "utils.h"
#include "types.h"
#define RSA_2048_BYTES 0x100
#define RSA_2048_BITS (RSA_2048_BYTES*8)
static void calculate_mgf1_and_xor(unsigned char *data, size_t data_size, const void *h_src, size_t h_src_size) {
unsigned char h_buf[RSA_2048_BYTES] = {0};
memcpy(h_buf, h_src, h_src_size);
unsigned char mgf1_buf[0x20];
size_t ofs = 0;
unsigned int seed = 0;
while (ofs < data_size) {
for (unsigned int i = 0; i < sizeof(seed); i++) {
h_buf[h_src_size + 3 - i] = (seed >> (8 * i)) & 0xFF;
}
sha256_hash_buffer(mgf1_buf, h_buf, h_src_size + 4);
for (unsigned int i = ofs; i < data_size && i < ofs + 0x20; i++) {
data[i] ^= mgf1_buf[i - ofs];
}
seed++;
ofs += 0x20;
}
}
/* Perform an RSA-PSS verify operation on data, with signature and N. */
int rsa2048_pss_verify(const void *data, size_t len, const unsigned char *signature, const unsigned char *modulus) {
mbedtls_mpi signature_mpi;
mbedtls_mpi modulus_mpi;
mbedtls_mpi e_mpi;
mbedtls_mpi message_mpi;
mbedtls_mpi_init(&signature_mpi);
mbedtls_mpi_init(&modulus_mpi);
mbedtls_mpi_init(&e_mpi);
mbedtls_mpi_init(&message_mpi);
mbedtls_mpi_lset(&message_mpi, RSA_2048_BITS);
unsigned char m_buf[RSA_2048_BYTES];
unsigned char h_buf[0x24];
const unsigned char E[3] = {1, 0, 1};
mbedtls_mpi_read_binary(&e_mpi, E, 3);
mbedtls_mpi_read_binary(&signature_mpi, signature, RSA_2048_BYTES);
mbedtls_mpi_read_binary(&modulus_mpi, modulus, RSA_2048_BYTES);
mbedtls_mpi_exp_mod(&message_mpi, &signature_mpi, &e_mpi, &modulus_mpi, NULL);
if (mbedtls_mpi_write_binary(&message_mpi, m_buf, RSA_2048_BYTES) != 0) {
FATAL_ERROR("Failed to export exponentiated RSA message!");
}
mbedtls_mpi_free(&signature_mpi);
mbedtls_mpi_free(&modulus_mpi);
mbedtls_mpi_free(&e_mpi);
mbedtls_mpi_free(&message_mpi);
/* There's no automated PSS verification as far as I can tell. */
if (m_buf[RSA_2048_BYTES-1] != 0xBC) {
return false;
}
memset(h_buf, 0, 0x24);
memcpy(h_buf, m_buf + RSA_2048_BYTES - 0x20 - 0x1, 0x20);
/* Decrypt maskedDB. */
calculate_mgf1_and_xor(m_buf, RSA_2048_BYTES - 0x20 - 1, h_buf, 0x20);
m_buf[0] &= 0x7F; /* Constant lmask for rsa-2048-pss. */
/* Validate DB. */
for (unsigned int i = 0; i < RSA_2048_BYTES - 0x20 - 0x20 - 1 - 1; i++) {
if (m_buf[i] != 0) {
return false;
}
}
if (m_buf[RSA_2048_BYTES - 0x20 - 0x20 - 1 - 1] != 1) {
return false;
}
/* Check hash correctness. */
unsigned char validate_buf[8 + 0x20 + 0x20];
unsigned char validate_hash[0x20];
memset(validate_buf, 0, 0x48);
sha256_hash_buffer(&validate_buf[8], data, len);
memcpy(&validate_buf[0x28], &m_buf[RSA_2048_BYTES - 0x20 - 0x20 - 1], 0x20);
sha256_hash_buffer(validate_hash, validate_buf, 0x48);
return memcmp(h_buf, validate_hash, 0x20) == 0;
}
/* Perform an RSA-PKCS1 verify operation on data, with signature and N. */
int rsa2048_pkcs1_verify(const void *data, size_t len, const unsigned char *signature, const unsigned char *modulus) {
mbedtls_mpi signature_mpi;
mbedtls_mpi modulus_mpi;
mbedtls_mpi e_mpi;
mbedtls_mpi message_mpi;
mbedtls_mpi_init(&signature_mpi);
mbedtls_mpi_init(&modulus_mpi);
mbedtls_mpi_init(&e_mpi);
mbedtls_mpi_init(&message_mpi);
mbedtls_mpi_lset(&message_mpi, RSA_2048_BITS);
unsigned char m_buf[RSA_2048_BYTES];
unsigned char h_buf[0x20];
const unsigned char E[3] = {1, 0, 1};
mbedtls_mpi_read_binary(&e_mpi, E, 3);
mbedtls_mpi_read_binary(&signature_mpi, signature, RSA_2048_BYTES);
mbedtls_mpi_read_binary(&modulus_mpi, modulus, RSA_2048_BYTES);
mbedtls_mpi_exp_mod(&message_mpi, &signature_mpi, &e_mpi, &modulus_mpi, NULL);
if (mbedtls_mpi_write_binary(&message_mpi, m_buf, RSA_2048_BYTES) != 0) {
FATAL_ERROR("Failed to export exponentiated RSA message!");
}
mbedtls_mpi_free(&signature_mpi);
mbedtls_mpi_free(&modulus_mpi);
mbedtls_mpi_free(&e_mpi);
mbedtls_mpi_free(&message_mpi);
/* For RSA-2048, this prefix is just a constant. */
const unsigned char pkcs1_hash_prefix[0xE0] = {
0x00, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x30, 0x31, 0x30,
0x0D, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20
};
sha256_hash_buffer(h_buf, data, len);
return memcmp(pkcs1_hash_prefix, m_buf, 0xE0) == 0 && memcmp(&m_buf[0xE0], h_buf, 0x20) == 0;
}
/* Perform an RSA-OAEP decryption (and verification) on data, with Signature and N. */
int rsa2048_oaep_decrypt_verify(void *out, size_t max_out_len, const unsigned char *signature, const unsigned char *modulus, const unsigned char *exponent, size_t exponent_len, const unsigned char *label_hash, size_t *out_len) {
mbedtls_mpi signature_mpi;
mbedtls_mpi modulus_mpi;
mbedtls_mpi exp_mpi;
mbedtls_mpi message_mpi;
mbedtls_mpi_init(&signature_mpi);
mbedtls_mpi_init(&modulus_mpi);
mbedtls_mpi_init(&exp_mpi);
mbedtls_mpi_init(&message_mpi);
mbedtls_mpi_lset(&message_mpi, RSA_2048_BITS);
unsigned char m_buf[RSA_2048_BYTES];
mbedtls_mpi_read_binary(&exp_mpi, exponent, exponent_len);
mbedtls_mpi_read_binary(&signature_mpi, signature, RSA_2048_BYTES);
mbedtls_mpi_read_binary(&modulus_mpi, modulus, RSA_2048_BYTES);
mbedtls_mpi_exp_mod(&message_mpi, &signature_mpi, &exp_mpi, &modulus_mpi, NULL);
if (mbedtls_mpi_write_binary(&message_mpi, m_buf, RSA_2048_BYTES) != 0) {
FATAL_ERROR("Failed to export exponentiated RSA message!");
}
mbedtls_mpi_free(&signature_mpi);
mbedtls_mpi_free(&modulus_mpi);
mbedtls_mpi_free(&exp_mpi);
mbedtls_mpi_free(&message_mpi);
/* There's no automated PSS verification as far as I can tell. */
if (m_buf[0] != 0x00) {
return false;
}
/* Unmask salt. */
calculate_mgf1_and_xor(m_buf + 1, 0x20, m_buf + 0x21, RSA_2048_BYTES - 0x20 - 1);
/* Unmask DB. */
calculate_mgf1_and_xor(m_buf + 0x21, RSA_2048_BYTES - 0x20 - 1, m_buf + 1, 0x20);
/* Validate label hash */
const unsigned char *db = m_buf + 0x21;
if (memcmp(db, label_hash, 0x20) != 0) {
return false;
}
/* Validate message prefix. */
const unsigned char *data = db + 0x20;
size_t remaining = RSA_2048_BYTES - 0x20 - 1 - 0x20;
while (*data == 0 && remaining) {
data++;
remaining--;
}
if (remaining == 0 || *data++ != 1) {
return false;
}
remaining--;
*out_len = remaining;
if (remaining > max_out_len) {
remaining = max_out_len;
}
memcpy(out, data, remaining);
return true;
}