Significantly improve preconv. Allow it to recode UTF-8 characters into

the \[uNNNN] strings (taking into account big-endian archs).  Also allow
it to determine from the BOM whether it's a UTF-8 file.  Also add the
initial manual.  This has been tested over a random selection of UTF-8
documents, as

 % preconv -e utf-8 foo.1 | ./mandoc -Tlocale

where -Tlocale is allowed (-DUSE_WCHAR).

Note that we're still missing the "type" indicator that preconv accepts.

3 files changed, 260 insertions, 12 deletions

diff --git a/Makefile b/Makefile
index aa3b29b7..5f139b8e 100644
--- a/Makefile
+++ b/Makefile
@@ -88,6 +88,8 @@ SRCS		 = Makefile \
 		   msec.in \
 		   out.c \
 		   out.h \
+		   preconv.1 \
+		   preconv.c \
 		   predefs.in \
 		   read.c \
 		   roff.7 \
@@ -268,6 +270,11 @@ INDEX_MANS	 = makewhatis.1.html \
 		   mdoc.7.ps \
 		   mdoc.7.pdf \
 		   mdoc.7.txt \
+		   preconv.1.html \
+		   preconv.1.xhtml \
+		   preconv.1.ps \
+		   preconv.1.pdf \
+		   preconv.1.txt \
 		   roff.7.html \
 		   roff.7.xhtml \
 		   roff.7.ps \
diff --git a/preconv.1 b/preconv.1
new file mode 100644
index 00000000..480897ef
--- /dev/null
+++ b/preconv.1
@@ -0,0 +1,137 @@
+.\"	$Id$
+.\"
+.\" Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
+.\"
+.\" Permission to use, copy, modify, and distribute this software for any
+.\" purpose with or without fee is hereby granted, provided that the above
+.\" copyright notice and this permission notice appear in all copies.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+.\"
+.Dd $Mdocdate$
+.Dt PRECONV 1
+.Os
+.Sh NAME
+.Nm preconv
+.Nd recodes multibyte UNIX manuals as mandoc input
+.Sh SYNOPSIS
+.Nm preconv
+.Op Fl D Ar enc
+.Op Fl e Ar enc
+.Op Ar file
+.Sh DESCRIPTION
+The
+.Nm
+utility recodes multibyte
+.Ux
+manual files into
+.Xr mandoc 1
+input.
+Its arguments are as follows:
+.Bl -tag -width Ds
+.It Fl D Ar enc
+The default encoding.
+This is case-insensitive.
+See
+.Sx Algorithm
+and
+.Sx Encodings .
+.It Fl e Ar enc
+The document's encoding.
+This is case-insensitive.
+See
+.Sx Algorithm
+and
+.Sx Encodings .
+.It Ar file
+The input file.
+.El
+.Pp
+If
+.Ar file
+is not provided,
+.Nm
+accepts standard input.
+Output is written to standard output.
+Unicode characters in the ASCII range are printed as regular ASCII
+characters; those above this range are printed using the
+.Sq \e[uNNNN]
+format documented in
+.Xr mandoc_char 7 .
+.Pp
+If input bytes are improperly formed in the current encoding, they're
+passed unmodified to standard output.
+.Ss Encodings
+The
+.Nm
+utility accepts the
+.Ar utf\-8 ,
+.Ar us\-ascii ,
+and
+.Ar latin\-1
+encodings as arguments to
+.Fl D Ar enc
+or
+.Fl e Ar enc .
+.Ss Algorithm
+An encoding is chosen according to the following steps:
+.Bl -enum
+.It
+From the argument passed to
+.Fl e Ar enc .
+.It
+If a BOM exists, utf\-8 encoding is selected.
+.It
+From the argument passed to
+.Fl D Ar enc .
+.It
+If all else fails, Latin\-1 is used.
+.El
+.\" .Sh IMPLEMENTATION NOTES
+.\" Not used in OpenBSD.
+.\" .Sh RETURN VALUES
+.\" For sections 2, 3, & 9 only.
+.\" .Sh ENVIRONMENT
+.\" For sections 1, 6, 7, & 8 only.
+.\" .Sh FILES
+.Sh EXIT STATUS
+.Ex -std
+.\" .Sh EXAMPLES
+.\" .Sh DIAGNOSTICS
+.\" For sections 1, 4, 6, 7, & 8 only.
+.\" .Sh ERRORS
+.\" For sections 2, 3, & 9 only.
+.Sh SEE ALSO
+.Xr mandoc 1 ,
+.Xr mandoc_char 7
+.Sh STANDARDS
+The
+.Nm
+utility references the US-ASCII character set standard, ANSI_X3.4\-1968;
+the Latin\-1 character set standard, ISO/IEC 8859\-1:1998; the UTF\-8
+character set standard; and UCS (Unicode), ISO/IEC 10646.
+.Sh HISTORY
+The
+.Nm
+utility first appeared in the GNU troff
+.Pq Dq groff
+system in December 2005, authored by Tomohiro Kubota and Werner
+Lemberg.
+The implementation that is part of the
+.Xr mandoc 1
+utility appeared in May 2011.
+.Sh AUTHORS
+The
+.Nm
+utility was written by
+.An Kristaps Dzonsons Aq kristaps@bsd.lv .
+.\" .Sh CAVEATS
+.\" .Sh BUGS
+.\" .Sh SECURITY CONSIDERATIONS
+.\" Not used in OpenBSD.
diff --git a/preconv.c b/preconv.c
index b39578d9..4bd49607 100644
--- a/preconv.c
+++ b/preconv.c
@@ -85,7 +85,7 @@ static int
 conv_latin_1(const struct buf *b)
 {
 	size_t		 i;
-	unsigned char	 c;
+	unsigned char	 cu;
 	const char	*cp;
 
 	cp = b->buf + (int)b->offs;
@@ -97,8 +97,8 @@ conv_latin_1(const struct buf *b)
 	 */
 
 	for (i = b->offs; i < b->sz; i++) {
-		c = (unsigned char)*cp++;
-		c < 128 ? putchar(c) : printf("\\[u%.4X]", c);
+		cu = (unsigned char)*cp++;
+		cu < 128U ? putchar(cu) : printf("\\[u%.4X]", cu);
 	}
 
 	return(1);
@@ -120,6 +120,105 @@ conv_us_ascii(const struct buf *b)
 static int
 conv_utf_8(const struct buf *b)
 {
+	int		 state, be;
+	unsigned int	 accum;
+	size_t		 i;
+	unsigned char	 cu;
+	const char	*cp;
+	const long	 one = 1L;
+
+	cp = b->buf + (int)b->offs;
+	state = 0;
+	accum = 0U;
+	be = 0;
+
+	/* Quick test for big-endian value. */
+
+	if ( ! (*((char *)(&one))))
+		be = 1;
+
+	for (i = b->offs; i < b->sz; i++) {
+		cu = (unsigned char)*cp++;
+		if (state) {
+			if ( ! (cu & 128) || (cu & 64)) {
+				/* Bad sequence header. */
+				return(0);
+			}
+
+			/* Accept only legitimate bit patterns. */
+
+			if (cu > 191 || cu < 128) {
+				/* Bad in-sequence bits. */
+				return(0);
+			}
+
+			accum |= (cu & 63) << --state * 6;
+
+			/*
+			 * Accum is held in little-endian order as
+			 * stipulated by the UTF-8 sequence coding.  We
+			 * need to convert to a native big-endian if our
+			 * architecture requires it.
+			 */
+
+			if (0 == state && be) 
+				accum = (accum >> 24) | 
+					((accum << 8) & 0x00FF0000) |
+					((accum >> 8) & 0x0000FF00) |
+					(accum << 24);
+
+			if (0 == state) {
+				accum < 128U ? putchar(accum) : 
+					printf("\\[u%.4X]", accum);
+				accum = 0U;
+			}
+		} else if (cu & (1 << 7)) {
+			/*
+			 * Entering a UTF-8 state:  if we encounter a
+			 * UTF-8 bitmask, calculate the expected UTF-8
+			 * state from it.
+			 */
+			for (state = 0; state < 7; state++) 
+				if ( ! (cu & (1 << (7 - state))))
+					break;
+
+			/* Accept only legitimate bit patterns. */
+
+			switch (state) {
+			case (4):
+				if (cu <= 244 && cu >= 240) {
+					accum = (cu & 7) << 18;
+					break;
+				}
+				/* Bad 4-sequence start bits. */
+				return(0);
+			case (3):
+				if (cu <= 239 && cu >= 224) {
+					accum = (cu & 15) << 12;
+					break;
+				}
+				/* Bad 3-sequence start bits. */
+				return(0);
+			case (2):
+				if (cu <= 223 && cu >= 194) {
+					accum = (cu & 31) << 6;
+					break;
+				}
+				/* Bad 2-sequence start bits. */
+				return(0);
+			default:
+				/* Bad sequence bit mask. */
+				return(0);
+			}
+			state--;
+		} else
+			putchar(cu);
+	}
+
+	if (0 != state) {
+		/* Bad trailing bits. */
+		return(0);
+	}
 
 	return(1);
 }
@@ -211,9 +310,10 @@ int
 main(int argc, char *argv[])
 {
 	int	 	 i, ch, map, fd, rc;
-	struct buf	 buf;
+	struct buf	 b;
 	const char	*fn;
 	enum enc	 enc, def;
+	const char	 bom[3] = { 0xEF, 0xBB, 0xBF };
 	extern int	 optind;
 	extern char	*optarg;
 
@@ -229,7 +329,7 @@ main(int argc, char *argv[])
 	enc = def = ENC__MAX;
 	map = 0;
 
-	memset(&buf, 0, sizeof(struct buf));
+	memset(&b, 0, sizeof(struct buf));
 
 	while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
 		switch (ch) {
@@ -283,12 +383,16 @@ main(int argc, char *argv[])
 		}
 	}
 
-	if ( ! read_whole_file(fn, fd, &buf, &map))
+	if ( ! read_whole_file(fn, fd, &b, &map))
 		goto out;
 
-	if (ENC__MAX == enc) {
-		/* TODO: search for BOM. */
-	}
+	/* Try to read the UTF-8 BOM. */
+
+	if (ENC__MAX == enc)
+		if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
+			b.offs = 3;
+			enc = ENC_UTF_8;
+		}
 
 	/*
 	 * No encoding has been detected.
@@ -299,15 +403,15 @@ main(int argc, char *argv[])
 	if (ENC__MAX == enc) 
 		enc = ENC__MAX == def ? ENC_LATIN_1 : def;
 
-	if ( ! (*encs[(int)enc].conv)(&buf))
+	if ( ! (*encs[(int)enc].conv)(&b)) 
 		goto out;
 
 	rc = EXIT_SUCCESS;
 out:
 	if (map)
-		munmap(buf.buf, buf.sz);
+		munmap(b.buf, b.sz);
 	else 
-		free(buf.buf);
+		free(b.buf);
 
 	if (fd > STDIN_FILENO)
 		close(fd);