1 /* $OpenBSD: encoding.c,v 1.13 2022/05/15 15:00:53 deraadt Exp $ */
2 /*
3 * Copyright (c) 2020 Claudio Jeker <claudio@openbsd.org>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17 #include <sys/stat.h>
18
19 #include <err.h>
20 #include <errno.h>
21 #include <ctype.h>
22 #include <fcntl.h>
23 #include <limits.h>
24 #include <stdlib.h>
25 #include <string.h>
26 #include <unistd.h>
27
28 #include <openssl/evp.h>
29
30 #include "extern.h"
31
32 /*
33 * Load file from disk and return the buffer and size.
34 */
35 unsigned char *
load_file(const char * name,size_t * len)36 load_file(const char *name, size_t *len)
37 {
38 unsigned char *buf = NULL;
39 struct stat st;
40 ssize_t n;
41 size_t size;
42 int fd, saved_errno;
43
44 *len = 0;
45
46 if ((fd = open(name, O_RDONLY)) == -1)
47 return NULL;
48 if (fstat(fd, &st) != 0)
49 goto err;
50 if (st.st_size <= 0 || st.st_size > MAX_FILE_SIZE) {
51 errno = EFBIG;
52 goto err;
53 }
54 size = (size_t)st.st_size;
55 if ((buf = malloc(size)) == NULL)
56 goto err;
57 n = read(fd, buf, size);
58 if (n == -1)
59 goto err;
60 if ((size_t)n != size) {
61 errno = EIO;
62 goto err;
63 }
64 close(fd);
65 *len = size;
66 return buf;
67
68 err:
69 saved_errno = errno;
70 close(fd);
71 free(buf);
72 errno = saved_errno;
73 return NULL;
74 }
75
76 /*
77 * Return the size of the data blob in outlen for an inlen sized base64 buffer.
78 * Returns 0 on success and -1 if inlen would overflow an int.
79 */
80 int
base64_decode_len(size_t inlen,size_t * outlen)81 base64_decode_len(size_t inlen, size_t *outlen)
82 {
83 *outlen = 0;
84 if (inlen >= INT_MAX - 3)
85 return -1;
86 *outlen = ((inlen + 3) / 4) * 3 + 1;
87 return 0;
88 }
89
90 /*
91 * Decode base64 encoded string into binary buffer returned in out.
92 * The out buffer size is stored in outlen.
93 * Returns 0 on success or -1 for any errors.
94 */
95 int
base64_decode(const unsigned char * in,size_t inlen,unsigned char ** out,size_t * outlen)96 base64_decode(const unsigned char *in, size_t inlen,
97 unsigned char **out, size_t *outlen)
98 {
99 EVP_ENCODE_CTX *ctx;
100 unsigned char *to = NULL;
101 size_t tolen;
102 int evplen;
103
104 if ((ctx = EVP_ENCODE_CTX_new()) == NULL)
105 err(1, "EVP_ENCODE_CTX_new");
106
107 *out = NULL;
108 *outlen = 0;
109
110 if (base64_decode_len(inlen, &tolen) == -1)
111 goto fail;
112 if ((to = malloc(tolen)) == NULL)
113 err(1, NULL);
114
115 evplen = tolen;
116 EVP_DecodeInit(ctx);
117 if (EVP_DecodeUpdate(ctx, to, &evplen, in, inlen) == -1)
118 goto fail;
119 *outlen = evplen;
120 if (EVP_DecodeFinal(ctx, to + evplen, &evplen) == -1)
121 goto fail;
122 *outlen += evplen;
123 *out = to;
124
125 EVP_ENCODE_CTX_free(ctx);
126 return 0;
127
128 fail:
129 free(to);
130 EVP_ENCODE_CTX_free(ctx);
131 return -1;
132 }
133
134 /*
135 * Return the size of the base64 blob in outlen for a inlen sized binary buffer.
136 * Returns 0 on success and -1 if inlen would overflow the calculation.
137 */
138 int
base64_encode_len(size_t inlen,size_t * outlen)139 base64_encode_len(size_t inlen, size_t *outlen)
140 {
141 *outlen = 0;
142 if (inlen >= INT_MAX / 2)
143 return -1;
144 *outlen = ((inlen + 2) / 3) * 4 + 1;
145 return 0;
146 }
147
148 /*
149 * Encode a binary buffer into a base64 encoded string returned in out.
150 * Returns 0 on success or -1 for any errors.
151 */
152 int
base64_encode(const unsigned char * in,size_t inlen,char ** out)153 base64_encode(const unsigned char *in, size_t inlen, char **out)
154 {
155 unsigned char *to;
156 size_t tolen;
157
158 *out = NULL;
159
160 if (base64_encode_len(inlen, &tolen) == -1)
161 return -1;
162 if ((to = malloc(tolen)) == NULL)
163 return -1;
164
165 EVP_EncodeBlock(to, in, inlen);
166 *out = to;
167 return 0;
168 }
169
170 /*
171 * Convert binary buffer of size dsz into an upper-case hex-string.
172 * Returns pointer to the newly allocated string. Function can't fail.
173 */
174 char *
hex_encode(const unsigned char * in,size_t insz)175 hex_encode(const unsigned char *in, size_t insz)
176 {
177 const char hex[] = "0123456789ABCDEF";
178 size_t i;
179 char *out;
180
181 if ((out = calloc(2, insz + 1)) == NULL)
182 err(1, NULL);
183
184 for (i = 0; i < insz; i++) {
185 out[i * 2] = hex[in[i] >> 4];
186 out[i * 2 + 1] = hex[in[i] & 0xf];
187 }
188 out[i * 2] = '\0';
189
190 return out;
191 }
192
193 /*
194 * Hex decode hexstring into the supplied buffer.
195 * Return 0 on success else -1, if buffer too small or bad encoding.
196 */
197 int
hex_decode(const char * hexstr,char * buf,size_t len)198 hex_decode(const char *hexstr, char *buf, size_t len)
199 {
200 unsigned char ch, r;
201 size_t pos = 0;
202 int i;
203
204 while (*hexstr) {
205 r = 0;
206 for (i = 0; i < 2; i++) {
207 ch = hexstr[i];
208 if (isdigit(ch))
209 ch -= '0';
210 else if (islower(ch))
211 ch -= ('a' - 10);
212 else if (isupper(ch))
213 ch -= ('A' - 10);
214 else
215 return -1;
216 if (ch > 0xf)
217 return -1;
218 r = r << 4 | ch;
219 }
220 if (pos < len)
221 buf[pos++] = r;
222 else
223 return -1;
224
225 hexstr += 2;
226 }
227 return 0;
228 }
229