dte test coverage

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 50.0% high: ≥ 85.0%
Coverage Exec / Excl / Total
Lines: 53.6% 162 / 2 / 304
Functions: 71.4% 20 / 0 / 28
Branches: 39.0% 46 / 18 / 136
src/convert.c
Line Branch Exec Source
1 #include <errno.h>
2 #include <inttypes.h>
3 #include <stdlib.h>
4 #include <string.h>
5 #include "convert.h"
6 #include "block.h"
7 #include "buildvar-iconv.h"
8 #include "encoding.h"
9 #include "util/arith.h"
10 #include "util/debug.h"
11 #include "util/list.h"
12 #include "util/log.h"
13 #include "util/str-util.h"
14 #include "util/utf8.h"
15 #include "util/xmalloc.h"
16 #include "util/xreadwrite.h"
17
18 typedef struct {
19 StringView text;
20 size_t ipos;
21 struct CharsetConverter *cconv;
22 } FileDecoder;
23
24 56 static void add_block(Buffer *buffer, Block *blk)
25 {
26 56 buffer->nl += blk->nl;
27 56 list_insert_before(&blk->node, &buffer->blocks);
28 56 }
29
30 7824 static Block *add_utf8_line(Buffer *buffer, Block *blk, StringView line)
31 {
32 7824 const size_t len = line.length;
33 7824 size_t size = len + 1;
34
2/2
✓ Branch 2 → 3 taken 7796 times.
✓ Branch 2 → 6 taken 28 times.
 7824 if (blk) {
35 7796 size_t avail = blk->alloc - blk->size;
36
2/2
✓ Branch 3 → 4 taken 7768 times.
✓ Branch 3 → 5 taken 28 times.
 7796 if (size <= avail) {
37 7768 goto copy;
38 }
39 28 add_block(buffer, blk);
40 }
41
42 56 size = MAX(size, 8192);
43 56 blk = block_new(size);
44
45 7824 copy:
46 7824 memcpy(blk->data + blk->size, line.data, len);
47 7824 blk->size += len;
48 7824 blk->data[blk->size++] = '\n';
49 7824 blk->nl++;
50 7824 return blk;
51 }
52
53 7858 static bool read_utf8_line(FileDecoder *dec, StringView *linep)
54 {
55 7858 size_t len = dec->text.length;
56
2/2
✓ Branch 2 → 3 taken 34 times.
✓ Branch 2 → 5 taken 7824 times.
 7858 if (dec->ipos >= len) {
57 34 BUG_ON(dec->ipos > len);
58 return false;
59 }
60
61 7824 *linep = get_delim(dec->text.data, &dec->ipos, len, '\n');
62 7824 return true;
63 }
64
65 34 static bool file_decoder_read_utf8(Buffer *buffer, StringView text, size_t *longest_line)
66 {
67
1/2
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 34 times.
 34 if (unlikely(!encoding_is_utf8(buffer->encoding))) {
68 ✗ errno = EINVAL;
69 ✗ return false;
70 }
71
72 34 FileDecoder dec = {.text = text};
73 34 StringView line;
74
2/2
✓ Branch 6 → 7 taken 6 times.
✓ Branch 6 → 8 taken 28 times.
 34 if (!read_utf8_line(&dec, &line)) {
75 6 *longest_line = 0;
76 6 return true;
77 }
78
79
2/2
✓ Branch 9 → 10 taken 1 time.
✓ Branch 9 → 11 taken 27 times.
 28 if (strview_remove_matching_suffix(&line, "\r")) {
80 1 buffer->crlf_newlines = true;
81 }
82
83 28 Block *blk = add_utf8_line(buffer, NULL, line);
84 28 size_t maxline = line.length;
85
86
2/2
✓ Branch 12 → 16 taken 1 time.
✓ Branch 12 → 20 taken 27 times.
 28 if (unlikely(buffer->crlf_newlines)) {
87
2/2
✓ Branch 17 → 13 taken 270 times.
✓ Branch 17 → 22 taken 1 time.
 271 while (read_utf8_line(&dec, &line)) {
88 270 strview_remove_matching_suffix(&line, "\r");
89 270 blk = add_utf8_line(buffer, blk, line);
90 270 maxline = MAX(maxline, line.length);
91 }
92 } else {
93
2/2
✓ Branch 21 → 18 taken 7526 times.
✓ Branch 21 → 22 taken 27 times.
 7553 while (read_utf8_line(&dec, &line)) {
94 7526 blk = add_utf8_line(buffer, blk, line);
95 7526 maxline = MAX(maxline, line.length);
96 }
97 }
98
99
1/2
✓ Branch 22 → 23 taken 28 times.
✗ Branch 22 → 24 not taken.
 28 if (blk) {
100 28 add_block(buffer, blk);
101 }
102
103 28 *longest_line = maxline;
104 28 return true;
105 }
106
107 1 static size_t unix_to_dos(FileEncoder *enc, StringView text, size_t nr_newlines)
108 {
109 1 BUG_ON(text.length && !strview_has_suffix(text, "\n")); // See sanity_check_blocks()
110 1 BUG_ON(nr_newlines > text.length);
111
112 1 const size_t new_len = text.length + nr_newlines;
113
1/2
✓ Branch 8 → 9 taken 1 time.
✗ Branch 8 → 12 not taken.
 1 if (enc->nsize < new_len) {
114 1 enc->nsize = xmul(text.length, 2);
115 1 enc->nbuf = xrealloc(enc->nbuf, enc->nsize);
116 }
117
118 1 size_t seen_nl = 0;
119 1 size_t dest_pos = 0;
120
121
2/2
✓ Branch 19 → 13 taken 3 times.
✓ Branch 19 → 20 taken 1 time.
 4 for (size_t src_pos = 0; src_pos < text.length; ) {
122 3 const char *src = text.data + src_pos;
123 3 char *dest = enc->nbuf + dest_pos;
124 3 char *end = memccpy(dest, src, '\n', text.length - src_pos);
125 3 BUG_ON(!end); // Loop condition prevents this
126
127 3 size_t line_len = (size_t)(end - dest);
128 3 src_pos += line_len;
129 3 BUG_ON(src_pos > text.length);
130
131 3 end[-1] = '\r';
132 3 end[0] = '\n';
133 3 dest_pos += line_len + 1;
134 3 seen_nl++;
135 }
136
137 1 BUG_ON(seen_nl != nr_newlines);
138 1 BUG_ON(dest_pos != new_len);
139 1 return dest_pos;
140 }
141
142 #if ICONV_DISABLE == 1 // iconv not available; use basic, UTF-8 implementation:
143
144 bool conversion_supported_by_iconv (
145 const char* UNUSED_ARG(from),
146 const char* UNUSED_ARG(to)
147 ) {
148 errno = EINVAL;
149 return false;
150 }
151
152 FileEncoder file_encoder(const char *encoding, bool crlf, int fd)
153 {
154 if (unlikely(!encoding_is_utf8(encoding))) {
155 BUG("unsupported conversion; should have been handled earlier");
156 }
157
158 return (FileEncoder) {
159 .crlf = crlf,
160 .fd = fd,
161 };
162 }
163
164 void file_encoder_free(FileEncoder *enc)
165 {
166 free(enc->nbuf);
167 }
168
169 ssize_t file_encoder_write (
170 FileEncoder *enc,
171 const char *buf,
172 size_t size,
173 size_t nr_newlines
174 ) {
175 if (unlikely(enc->crlf)) {
176 size = unix_to_dos(enc, string_view(buf, size), nr_newlines);
177 buf = enc->nbuf;
178 }
179 return xwrite_all(enc->fd, buf, size);
180 }
181
182 size_t file_encoder_get_nr_errors(const FileEncoder* UNUSED_ARG(enc))
183 {
184 return 0;
185 }
186
187 bool file_decoder_read(Buffer *buffer, StringView text, size_t *longest_line)
188 {
189 return file_decoder_read_utf8(buffer, text, longest_line);
190 }
191
192 #else // ICONV_DISABLE != 1; use full iconv implementation:
193
194 #include <iconv.h>
195
196 // UTF-8 encoding of U+00BF (inverted question mark; "¿")
197 #define REPLACEMENT "\xc2\xbf"
198
199 typedef struct CharsetConverter {
200 iconv_t cd;
201 char *obuf;
202 size_t osize;
203 size_t opos;
204 size_t consumed;
205 size_t errors;
206
207 // Temporary input buffer
208 char tbuf[16];
209 size_t tcount;
210
211 // REPLACEMENT character, in target encoding
212 char rbuf[4];
213 size_t rcount;
214
215 // Input character size in bytes, or zero for UTF-8
216 size_t char_size;
217 } CharsetConverter;
218
219 1 static CharsetConverter *create(iconv_t cd)
220 {
221 1 CharsetConverter *c = xcalloc1(sizeof(*c));
222 1 c->cd = cd;
223 1 c->osize = 8192;
224 1 c->obuf = xmalloc(c->osize);
225 1 return c;
226 }
227
228 2 static size_t iconv_wrapper (
229 iconv_t cd,
230 const char **restrict inbuf,
231 size_t *restrict inbytesleft,
232 char **restrict outbuf,
233 size_t *restrict outbytesleft
234 ) {
235 // POSIX defines the second parameter of iconv(3) as "char **restrict"
236 // but NetBSD declares it as "const char **restrict"
237 #ifdef __NetBSD__
238 const char **restrict in = inbuf;
239 #else
240 2 char **restrict in = (char **restrict)inbuf;
241 #endif
242
243 2 return iconv(cd, in, inbytesleft, outbuf, outbytesleft);
244 }
245
246 ✗ static void resize_obuf(CharsetConverter *c)
247 {
248 ✗ c->osize = xmul(2, c->osize);
249 ✗ c->obuf = xrealloc(c->obuf, c->osize);
250 ✗ }
251
252 ✗ static void add_replacement(CharsetConverter *c)
253 {
254 ✗ if (c->osize - c->opos < 4) {
255 ✗ resize_obuf(c);
256 }
257
258 ✗ memcpy(c->obuf + c->opos, c->rbuf, c->rcount);
259 ✗ c->opos += c->rcount;
260 ✗ }
261
262 ✗ static size_t handle_invalid(CharsetConverter *c, const char *buf, size_t count)
263 {
264 ✗ LOG_DEBUG("%zu %zu", c->char_size, count);
265 ✗ add_replacement(c);
266 ✗ if (c->char_size == 0) {
267 // Converting from UTF-8
268 ✗ size_t idx = 0;
269 ✗ CodePoint u = u_get_char(buf, count, &idx);
270 ✗ LOG_DEBUG("U+%04" PRIX32, u);
271 ✗ return idx;
272 }
273 ✗ if (c->char_size > count) {
274 // wtf
275 ✗ return 1;
276 }
277 return c->char_size;
278 }
279
280 1 static int xiconv(CharsetConverter *c, const char **ib, size_t *ic)
281 {
282 1 while (1) {
283 1 char *ob = c->obuf + c->opos;
284 1 size_t oc = c->osize - c->opos;
285 1 size_t rc = iconv_wrapper(c->cd, ib, ic, &ob, &oc);
286 1 c->opos = ob - c->obuf;
287
1/2
✗ Branch 4 → 5 not taken.
✓ Branch 4 → 12 taken 1 time.
 1 if (rc == (size_t)-1) {
288 ✗ switch (errno) {
289 ✗ case EILSEQ:
290 ✗ c->errors++;
291 // Reset
292 ✗ iconv(c->cd, NULL, NULL, NULL, NULL);
293 ✗ return errno;
294 case EINVAL:
295 return errno;
296 ✗ case E2BIG:
297 ✗ resize_obuf(c);
298 ✗ continue;
299 ✗ default:
300 − BUG("iconv: %s", strerror(errno));
301 }
302 } else {
303 1 c->errors += rc;
304 }
305 1 return 0;
306 }
307 }
308
309 ✗ static size_t convert_incomplete(CharsetConverter *c, const char *input, size_t len)
310 {
311 ✗ size_t ipos = 0;
312 ✗ while (c->tcount < sizeof(c->tbuf) && ipos < len) {
313 ✗ c->tbuf[c->tcount++] = input[ipos++];
314 ✗ const char *ib = c->tbuf;
315 ✗ size_t ic = c->tcount;
316 ✗ int rc = xiconv(c, &ib, &ic);
317 ✗ if (ic > 0) {
318 ✗ memmove(c->tbuf, ib, ic);
319 }
320 ✗ c->tcount = ic;
321 ✗ if (rc == EINVAL) {
322 // Incomplete character at end of input buffer; try again
323 // with more input data
324 ✗ continue;
325 }
326 ✗ if (rc == EILSEQ) {
327 // Invalid multibyte sequence
328 ✗ size_t skip = handle_invalid(c, c->tbuf, c->tcount);
329 ✗ c->tcount -= skip;
330 ✗ if (c->tcount > 0) {
331 ✗ LOG_DEBUG("tcount=%zu, skip=%zu", c->tcount, skip);
332 ✗ memmove(c->tbuf, c->tbuf + skip, c->tcount);
333 ✗ continue;
334 }
335 ✗ return ipos;
336 }
337 ✗ break;
338 }
339
340 ✗ LOG_DEBUG("%zu %zu", ipos, c->tcount);
341 ✗ return ipos;
342 }
343
344 1 static void cconv_process(CharsetConverter *c, const char *input, size_t len)
345 {
346
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 1 time.
 1 if (c->consumed > 0) {
347 ✗ size_t fill = c->opos - c->consumed;
348 ✗ memmove(c->obuf, c->obuf + c->consumed, fill);
349 ✗ c->opos = fill;
350 ✗ c->consumed = 0;
351 }
352
353
1/2
✗ Branch 4 → 5 not taken.
✓ Branch 4 → 7 taken 1 time.
 1 if (c->tcount > 0) {
354 ✗ size_t ipos = convert_incomplete(c, input, len);
355 ✗ input += ipos;
356 ✗ len -= ipos;
357 }
358
359 1 const char *ib = input;
360
2/2
✓ Branch 17 → 8 taken 1 time.
✓ Branch 17 → 18 taken 1 time.
 2 for (size_t ic = len; ic > 0; ) {
361 1 int r = xiconv(c, &ib, &ic);
362
1/2
✗ Branch 9 → 10 not taken.
✓ Branch 9 → 13 taken 1 time.
 1 if (r == EINVAL) {
363 // Incomplete character at end of input buffer
364 ✗ if (ic < sizeof(c->tbuf)) {
365 ✗ memcpy(c->tbuf, ib, ic);
366 ✗ c->tcount = ic;
367 } else {
368 // FIXME
369 ✗ }
370 ✗ ic = 0;
371 ✗ continue;
372 }
373
1/2
✗ Branch 13 → 14 not taken.
✓ Branch 13 → 16 taken 1 time.
 1 if (r == EILSEQ) {
374 // Invalid multibyte sequence
375 ✗ size_t skip = handle_invalid(c, ib, ic);
376 ✗ ic -= skip;
377 ✗ ib += skip;
378 ✗ continue;
379 }
380 }
381 1 }
382
383 ✗ static CharsetConverter *cconv_to_utf8(const char *encoding)
384 {
385 ✗ iconv_t cd = iconv_open("UTF-8", encoding);
386 ✗ if (cd == (iconv_t)-1) {
387 return NULL;
388 }
389
390 ✗ CharsetConverter *c = create(cd);
391 ✗ c->rcount = copyliteral(c->rbuf, REPLACEMENT);
392
393 ✗ if (str_has_prefix(encoding, "UTF-16")) {
394 ✗ c->char_size = 2;
395 ✗ } else if (str_has_prefix(encoding, "UTF-32")) {
396 ✗ c->char_size = 4;
397 } else {
398 ✗ c->char_size = 1;
399 }
400
401 return c;
402 }
403
404 1 static void encode_replacement(CharsetConverter *c)
405 {
406 1 static const char rep[] = REPLACEMENT;
407 1 const char *ib = rep;
408 1 char *ob = c->rbuf;
409 1 size_t ic = STRLEN(REPLACEMENT);
410 1 size_t oc = sizeof(c->rbuf);
411 1 size_t rc = iconv_wrapper(c->cd, &ib, &ic, &ob, &oc);
412
413
1/2
✓ Branch 3 → 4 taken 1 time.
✗ Branch 3 → 5 not taken.
 1 if (rc == (size_t)-1) {
414 1 c->rbuf[0] = '\xbf';
415 1 c->rcount = 1;
416 } else {
417 ✗ c->rcount = ob - c->rbuf;
418 }
419 1 }
420
421 1 static CharsetConverter *cconv_from_utf8(const char *encoding)
422 {
423 1 iconv_t cd = iconv_open(encoding, "UTF-8");
424
1/2
✓ Branch 3 → 4 taken 1 time.
✗ Branch 3 → 7 not taken.
 1 if (cd == (iconv_t)-1) {
425 return NULL;
426 }
427 1 CharsetConverter *c = create(cd);
428 1 encode_replacement(c);
429 1 return c;
430 }
431
432 1 static void cconv_flush(CharsetConverter *c)
433 {
434
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 6 taken 1 time.
 1 if (c->tcount > 0) {
435 // Replace incomplete character at end of input buffer
436 ✗ LOG_DEBUG("incomplete character at EOF");
437 ✗ add_replacement(c);
438 ✗ c->tcount = 0;
439 }
440 1 }
441
442 ✗ static char *cconv_consume_line(CharsetConverter *c, size_t *len)
443 {
444 ✗ char *line = c->obuf + c->consumed;
445 ✗ char *nl = memchr(line, '\n', c->opos - c->consumed);
446 ✗ if (!nl) {
447 ✗ *len = 0;
448 ✗ return NULL;
449 }
450
451 ✗ size_t n = nl - line + 1;
452 ✗ c->consumed += n;
453 ✗ *len = n;
454 ✗ return line;
455 }
456
457 1 static char *cconv_consume_all(CharsetConverter *c, size_t *len)
458 {
459 1 char *buf = c->obuf + c->consumed;
460 1 *len = c->opos - c->consumed;
461 1 c->consumed = c->opos;
462 1 return buf;
463 }
464
465 1 static void cconv_free(CharsetConverter *c)
466 {
467 1 BUG_ON(!c);
468 1 iconv_close(c->cd);
469 1 free(c->obuf);
470 1 free(c);
471 1 }
472
473 2 bool conversion_supported_by_iconv(const char *from, const char *to)
474 {
475
2/4
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 4 not taken.
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 2 times.
 2 if (unlikely(from[0] == '\0' || to[0] == '\0')) {
476 ✗ errno = EINVAL;
477 ✗ return false;
478 }
479
480 2 iconv_t cd = iconv_open(to, from);
481
1/2
✓ Branch 6 → 7 taken 2 times.
✗ Branch 6 → 9 not taken.
 2 if (cd == (iconv_t)-1) {
482 return false;
483 }
484
485 2 iconv_close(cd);
486 2 return true;
487 }
488
489 22 FileEncoder file_encoder(const char *encoding, bool crlf, int fd)
490 {
491 22 CharsetConverter *cconv = NULL;
492
2/2
✓ Branch 3 → 4 taken 1 time.
✓ Branch 3 → 7 taken 21 times.
 22 if (unlikely(!encoding_is_utf8(encoding))) {
493 1 cconv = cconv_from_utf8(encoding);
494
1/2
✗ Branch 5 → 6 not taken.
✓ Branch 5 → 7 taken 1 time.
 1 if (!cconv) {
495 − BUG("unsupported conversion; should have been handled earlier");
496 }
497 }
498
499 22 return (FileEncoder) {
500 .cconv = cconv,
501 .crlf = crlf,
502 .fd = fd,
503 };
504 }
505
506 22 void file_encoder_free(FileEncoder *enc)
507 {
508
2/2
✓ Branch 2 → 3 taken 1 time.
✓ Branch 2 → 4 taken 21 times.
 22 if (enc->cconv) {
509 1 cconv_free(enc->cconv);
510 }
511 22 free(enc->nbuf);
512 22 }
513
514 // NOTE: buf must contain whole characters!
515 22 ssize_t file_encoder_write (
516 FileEncoder *enc,
517 const char *buf,
518 size_t size,
519 size_t nr_newlines
520 ) {
521
2/2
✓ Branch 2 → 3 taken 1 time.
✓ Branch 2 → 5 taken 21 times.
 22 if (unlikely(enc->crlf)) {
522 1 size = unix_to_dos(enc, string_view(buf, size), nr_newlines);
523 1 buf = enc->nbuf;
524 }
525
2/2
✓ Branch 5 → 6 taken 1 time.
✓ Branch 5 → 9 taken 21 times.
 22 if (unlikely(enc->cconv)) {
526 1 cconv_process(enc->cconv, buf, size);
527 1 cconv_flush(enc->cconv);
528 1 buf = cconv_consume_all(enc->cconv, &size);
529 }
530 22 return xwrite_all(enc->fd, buf, size);
531 }
532
533 22 size_t file_encoder_get_nr_errors(const FileEncoder *enc)
534 {
535
2/2
✓ Branch 2 → 3 taken 1 time.
✓ Branch 2 → 4 taken 21 times.
 22 return enc->cconv ? enc->cconv->errors : 0;
536 }
537
538 ✗ static bool fill(FileDecoder *dec)
539 {
540 ✗ StringView text = dec->text;
541 ✗ if (dec->ipos == text.length) {
542 return false;
543 }
544
545 // Smaller than cconv.obuf to make realloc less likely
546 ✗ size_t max = 7 * 1024;
547
548 ✗ size_t icount = MIN(text.length - dec->ipos, max);
549 ✗ cconv_process(dec->cconv, text.data + dec->ipos, icount);
550 ✗ dec->ipos += icount;
551 ✗ if (dec->ipos == text.length) {
552 // Must be flushed after all input has been fed
553 ✗ cconv_flush(dec->cconv);
554 }
555 return true;
556 }
557
558 ✗ static bool decode_and_read_line(FileDecoder *dec, StringView *linep)
559 {
560 ✗ char *line;
561 ✗ size_t len;
562 ✗ while (1) {
563 ✗ line = cconv_consume_line(dec->cconv, &len);
564 ✗ if (line || !fill(dec)) {
565 break;
566 }
567 }
568
569 ✗ if (line) {
570 // Newline not wanted
571 ✗ len--;
572 } else {
573 ✗ line = cconv_consume_all(dec->cconv, &len);
574 ✗ if (len == 0) {
575 return false;
576 }
577 }
578
579 ✗ *linep = string_view(line, len);
580 ✗ return true;
581 }
582
583 34 bool file_decoder_read(Buffer *buffer, StringView text, size_t *longest_line)
584 {
585
1/2
✓ Branch 3 → 4 taken 34 times.
✗ Branch 3 → 5 not taken.
 34 if (encoding_is_utf8(buffer->encoding)) {
586 34 return file_decoder_read_utf8(buffer, text, longest_line);
587 }
588
589 ✗ CharsetConverter *cconv = cconv_to_utf8(buffer->encoding);
590 ✗ if (!cconv) {
591 return false;
592 }
593
594 ✗ FileDecoder dec = {.text = text, .cconv = cconv};
595 ✗ StringView line;
596 ✗ if (!decode_and_read_line(&dec, &line)) {
597 ✗ *longest_line = 0;
598 ✗ cconv_free(cconv);
599 ✗ return true;
600 }
601
602 ✗ if (strview_remove_matching_suffix(&line, "\r")) {
603 ✗ buffer->crlf_newlines = true;
604 }
605
606 ✗ Block *blk = add_utf8_line(buffer, NULL, line);
607 ✗ size_t maxline = line.length;
608 ✗ while (decode_and_read_line(&dec, &line)) {
609 ✗ if (buffer->crlf_newlines) {
610 ✗ strview_remove_matching_suffix(&line, "\r");
611 }
612 ✗ blk = add_utf8_line(buffer, blk, line);
613 ✗ maxline = MAX(maxline, line.length);
614 }
615
616 ✗ if (blk) {
617 ✗ add_block(buffer, blk);
618 }
619
620 ✗ *longest_line = maxline;
621 ✗ cconv_free(cconv);
622 ✗ return true;
623 }
624
625 #endif
626