path: root/preconv.c

/*	$Id$ */
/*
 * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <sys/stat.h>
#include <sys/mman.h>

#include <assert.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

/* 
 * The read_whole_file() and resize_buf() functions are copied from
 * read.c, including all dependency code (MAP_FILE, etc.).
 */

#ifndef MAP_FILE
#define	MAP_FILE	0
#endif

enum	enc {
	ENC_UTF_8, /* UTF-8 */
	ENC_US_ASCII, /* US-ASCII */
	ENC_LATIN_1, /* Latin-1 */
	ENC__MAX
};

struct	buf {
	char		 *buf; /* binary input buffer */
	size_t	 	  sz; /* size of binary buffer */
	size_t		  offs; /* starting buffer offset */
};

struct	encode {
	const char	 *name;
	int		(*conv)(const struct buf *);
};

static	int	 cue_enc(const struct buf *, size_t *, enum enc *);
static	int	 conv_latin_1(const struct buf *);
static	int	 conv_us_ascii(const struct buf *);
static	int	 conv_utf_8(const struct buf *);
static	int	 read_whole_file(const char *, int, 
			struct buf *, int *);
static	void	 resize_buf(struct buf *, size_t);
static	void	 usage(void);

static	const struct encode encs[ENC__MAX] = {
	{ "utf-8", conv_utf_8 }, /* ENC_UTF_8 */
	{ "us-ascii", conv_us_ascii }, /* ENC_US_ASCII */
	{ "latin-1", conv_latin_1 }, /* ENC_LATIN_1 */
};

static	const char	 *progname;

static void
usage(void)
{

	fprintf(stderr, "usage: %s "
			"[-D enc] "
			"[-e ENC] "
			"[file]\n", progname);
}

static int
conv_latin_1(const struct buf *b)
{
	size_t		 i;
	unsigned char	 cu;
	const char	*cp;

	cp = b->buf + (int)b->offs;

	/*
	 * Latin-1 falls into the first 256 code-points of Unicode, so
	 * there's no need for any sort of translation.  Just make the
	 * 8-bit characters use the Unicode escape.
	 * Note that binary values 128 < v < 160 are passed through
	 * unmodified to mandoc.
	 */

	for (i = b->offs; i < b->sz; i++) {
		cu = (unsigned char)*cp++;
		cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu);
	}

	return(1);
}

static int
conv_us_ascii(const struct buf *b)
{

	/*
	 * US-ASCII has no conversion since it falls into the first 128
	 * bytes of Unicode.
	 */

	fwrite(b->buf, 1, b->sz, stdout);
	return(1);
}

static int
conv_utf_8(const struct buf *b)
{
	int		 state, be;
	unsigned int	 accum;
	size_t		 i;
	unsigned char	 cu;
	const char	*cp;
	const long	 one = 1L;

	cp = b->buf + (int)b->offs;
	state = 0;
	accum = 0U;
	be = 0;

	/* Quick test for big-endian value. */

	if ( ! (*((const char *)(&one))))
		be = 1;

	for (i = b->offs; i < b->sz; i++) {
		cu = (unsigned char)*cp++;
		if (state) {
			if ( ! (cu & 128) || (cu & 64)) {
				/* Bad sequence header. */
				return(0);
			}

			/* Accept only legitimate bit patterns. */

			if (cu > 191 || cu < 128) {
				/* Bad in-sequence bits. */
				return(0);
			}

			accum |= (cu & 63) << --state * 6;

			/*
			 * Accum is held in little-endian order as
			 * stipulated by the UTF-8 sequence coding.  We
			 * need to convert to a native big-endian if our
			 * architecture requires it.
			 */

			if (0 == state && be) 
				accum = (accum >> 24) | 
					((accum << 8) & 0x00FF0000) |
					((accum >> 8) & 0x0000FF00) |
					(accum << 24);

			if (0 == state) {
				accum < 128U ? putchar(accum) : 
					printf("\\[u%.4X]", accum);
				accum = 0U;
			}
		} else if (cu & (1 << 7)) {
			/*
			 * Entering a UTF-8 state:  if we encounter a
			 * UTF-8 bitmask, calculate the expected UTF-8
			 * state from it.
			 */
			for (state = 0; state < 7; state++) 
				if ( ! (cu & (1 << (7 - state))))
					break;

			/* Accept only legitimate bit patterns. */

			switch (state) {
			case (4):
				if (cu <= 244 && cu >= 240) {
					accum = (cu & 7) << 18;
					break;
				}
				/* Bad 4-sequence start bits. */
				return(0);
			case (3):
				if (cu <= 239 && cu >= 224) {
					accum = (cu & 15) << 12;
					break;
				}
				/* Bad 3-sequence start bits. */
				return(0);
			case (2):
				if (cu <= 223 && cu >= 194) {
					accum = (cu & 31) << 6;
					break;
				}
				/* Bad 2-sequence start bits. */
				return(0);
			default:
				/* Bad sequence bit mask. */
				return(0);
			}
			state--;
		} else
			putchar(cu);
	}

	if (0 != state) {
		/* Bad trailing bits. */
		return(0);
	}

	return(1);
}

static void
resize_buf(struct buf *buf, size_t initial)
{

	buf->sz = buf->sz > initial / 2 ? 
		2 * buf->sz : initial;

	buf->buf = realloc(buf->buf, buf->sz);
	if (NULL == buf->buf) {
		perror(NULL);
		exit(EXIT_FAILURE);
	}
}

static int
read_whole_file(const char *f, int fd, 
		struct buf *fb, int *with_mmap)
{
	struct stat	 st;
	size_t		 off;
	ssize_t		 ssz;

	if (-1 == fstat(fd, &st)) {
		perror(f);
		return(0);
	}

	/*
	 * If we're a regular file, try just reading in the whole entry
	 * via mmap().  This is faster than reading it into blocks, and
	 * since each file is only a few bytes to begin with, I'm not
	 * concerned that this is going to tank any machines.
	 */

	if (S_ISREG(st.st_mode) && st.st_size >= (1U << 31)) {
		fprintf(stderr, "%s: input too large\n", f);
		return(0);
	} 
	
	if (S_ISREG(st.st_mode)) {
		*with_mmap = 1;
		fb->sz = (size_t)st.st_size;
		fb->buf = mmap(NULL, fb->sz, PROT_READ, 
				MAP_FILE|MAP_SHARED, fd, 0);
		if (fb->buf != MAP_FAILED)
			return(1);
	}

	/*
	 * If this isn't a regular file (like, say, stdin), then we must
	 * go the old way and just read things in bit by bit.
	 */

	*with_mmap = 0;
	off = 0;
	fb->sz = 0;
	fb->buf = NULL;
	for (;;) {
		if (off == fb->sz && fb->sz == (1U << 31)) {
			fprintf(stderr, "%s: input too large\n", f);
			break;
		} 
		
		if (off == fb->sz)
			resize_buf(fb, 65536);

		ssz = read(fd, fb->buf + (int)off, fb->sz - off);
		if (ssz == 0) {
			fb->sz = off;
			return(1);
		}
		if (ssz == -1) {
			perror(f);
			break;
		}
		off += (size_t)ssz;
	}

	free(fb->buf);
	fb->buf = NULL;
	return(0);
}

static int
cue_enc(const struct buf *b, size_t *offs, enum enc *enc)
{
	const char	*ln, *eoln, *eoph;
	size_t		 sz, phsz, nsz;
	int		 i;

	ln = b->buf + (int)*offs;
	sz = b->sz - *offs;

	/* Look for the end-of-line. */

	if (NULL == (eoln = memchr(ln, '\n', sz)))
		return(-1);

	/* Set next-line marker. */

	*offs = (size_t)((eoln + 1) - b->buf);

	/* Check if we have the correct header/trailer. */

	if ((sz = (size_t)(eoln - ln)) < 10 || 
			memcmp(ln, ".\\\" -*-", 7) ||
			memcmp(eoln - 3, "-*-", 3))
		return(0);

	/* Move after the header and adjust for the trailer. */

	ln += 7;
	sz -= 10;

	while (sz > 0) {
		while (sz > 0 && ' ' == *ln) {
			ln++;
			sz--;
		}
		if (0 == sz)
			break;

		/* Find the end-of-phrase marker (or eoln). */

		if (NULL == (eoph = memchr(ln, ';', sz)))
			eoph = eoln - 3;
		else
			eoph++;

		/* Only account for the "coding" phrase. */

		if ((phsz = (size_t)(eoph - ln)) < 7 ||
				strncasecmp(ln, "coding:", 7)) {
			sz -= phsz;
			ln += phsz;
			continue;
		} 

		sz -= 7;
		ln += 7;

		while (sz > 0 && ' ' == *ln) {
			ln++;
			sz--;
		}
		if (0 == sz)
			break;

		/* Check us against known encodings. */

		for (i = 0; i < (int)ENC__MAX; i++) {
			nsz = strlen(encs[i].name);
			if (phsz < nsz)
				continue;
			if (strncasecmp(ln, encs[i].name, nsz))
				continue;

			*enc = (enum enc)i;
			return(1);
		}

		/* Unknown encoding. */

		*enc = ENC__MAX;
		return(1);
	}

	return(0);
}

int
main(int argc, char *argv[])
{
	int	 	 i, ch, map, fd, rc;
	struct buf	 b;
	const char	*fn;
	enum enc	 enc, def;
	unsigned char 	 bom[3] = { 0xEF, 0xBB, 0xBF };
	size_t		 offs;
	extern int	 optind;
	extern char	*optarg;

	progname = strrchr(argv[0], '/');
	if (progname == NULL)
		progname = argv[0];
	else
		++progname;

	fn = "<stdin>";
	fd = STDIN_FILENO;
	rc = EXIT_FAILURE;
	enc = def = ENC__MAX;
	map = 0;

	memset(&b, 0, sizeof(struct buf));

	while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
		switch (ch) {
		case ('D'):
			/* FALLTHROUGH */
		case ('e'):
			for (i = 0; i < (int)ENC__MAX; i++) {
				if (strcasecmp(optarg, encs[i].name))
					continue;
				break;
			}
			if (i < (int)ENC__MAX) {
				if ('D' == ch)
					def = (enum enc)i;
				else
					enc = (enum enc)i;
				break;
			}

			fprintf(stderr, "%s: Bad encoding\n", optarg);
			return(EXIT_FAILURE);
		case ('r'):
			/* FALLTHROUGH */
		case ('d'):
			/* FALLTHROUGH */
		case ('v'):
			/* Compatibility with GNU preconv. */
			break;
		case ('h'):
			/* Compatibility with GNU preconv. */
			/* FALLTHROUGH */
		default:
			usage();
			return(EXIT_FAILURE);
		}

	argc -= optind;
	argv += optind;
	
	/* 
	 * Open and read the first argument on the command-line.
	 * If we don't have one, we default to stdin.
	 */

	if (argc > 0) {
		fn = *argv;
		fd = open(fn, O_RDONLY, 0);
		if (-1 == fd) {
			perror(fn);
			return(EXIT_FAILURE);
		}
	}

	if ( ! read_whole_file(fn, fd, &b, &map))
		goto out;

	/* Try to read the UTF-8 BOM. */

	if (ENC__MAX == enc)
		if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
			b.offs = 3;
			enc = ENC_UTF_8;
		}

	/* Try reading from the "-*-" cue. */

	if (ENC__MAX == enc) {
		offs = b.offs;
		ch = cue_enc(&b, &offs, &enc);
		if (0 == ch)
			ch = cue_enc(&b, &offs, &enc);
	}

	/*
	 * No encoding has been detected.
	 * Thus, we either fall into our default encoder, if specified,
	 * or use Latin-1 if all else fails.
	 */

	if (ENC__MAX == enc) 
		enc = ENC__MAX == def ? ENC_LATIN_1 : def;

	if ( ! (*encs[(int)enc].conv)(&b)) {
		fprintf(stderr, "%s: Bad encoding\n", fn);
		goto out;
	}

	rc = EXIT_SUCCESS;
out:
	if (map)
		munmap(b.buf, b.sz);
	else 
		free(b.buf);

	if (fd > STDIN_FILENO)
		close(fd);

	return(rc);
}