summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKristaps Dzonsons <kristaps@bsd.lv>2011-05-26 12:01:14 +0000
committerKristaps Dzonsons <kristaps@bsd.lv>2011-05-26 12:01:14 +0000
commitdc53a158a4c500cecef7c2188607bd0b8b5057fc (patch)
treed908e526812a0be8966b53631b42b10cf2e3b6e1
parent64e5d433ae9c2c8f8033d21ee8ec22eaeafe1dcd (diff)
downloadmandoc-dc53a158a4c500cecef7c2188607bd0b8b5057fc.tar.gz
Significantly improve preconv. Allow it to recode UTF-8 characters into
the \[uNNNN] strings (taking into account big-endian archs). Also allow it to determine from the BOM whether it's a UTF-8 file. Also add the initial manual. This has been tested over a random selection of UTF-8 documents, as % preconv -e utf-8 foo.1 | ./mandoc -Tlocale where -Tlocale is allowed (-DUSE_WCHAR). Note that we're still missing the "type" indicator that preconv accepts.
-rw-r--r--Makefile7
-rw-r--r--preconv.1137
-rw-r--r--preconv.c128
3 files changed, 260 insertions, 12 deletions
diff --git a/Makefile b/Makefile
index aa3b29b7..5f139b8e 100644
--- a/Makefile
+++ b/Makefile
@@ -88,6 +88,8 @@ SRCS = Makefile \
msec.in \
out.c \
out.h \
+ preconv.1 \
+ preconv.c \
predefs.in \
read.c \
roff.7 \
@@ -268,6 +270,11 @@ INDEX_MANS = makewhatis.1.html \
mdoc.7.ps \
mdoc.7.pdf \
mdoc.7.txt \
+ preconv.1.html \
+ preconv.1.xhtml \
+ preconv.1.ps \
+ preconv.1.pdf \
+ preconv.1.txt \
roff.7.html \
roff.7.xhtml \
roff.7.ps \
diff --git a/preconv.1 b/preconv.1
new file mode 100644
index 00000000..480897ef
--- /dev/null
+++ b/preconv.1
@@ -0,0 +1,137 @@
+.\" $Id$
+.\"
+.\" Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
+.\"
+.\" Permission to use, copy, modify, and distribute this software for any
+.\" purpose with or without fee is hereby granted, provided that the above
+.\" copyright notice and this permission notice appear in all copies.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+.\"
+.Dd $Mdocdate$
+.Dt PRECONV 1
+.Os
+.Sh NAME
+.Nm preconv
+.Nd recodes multibyte UNIX manuals as mandoc input
+.Sh SYNOPSIS
+.Nm preconv
+.Op Fl D Ar enc
+.Op Fl e Ar enc
+.Op Ar file
+.Sh DESCRIPTION
+The
+.Nm
+utility recodes multibyte
+.Ux
+manual files into
+.Xr mandoc 1
+input.
+Its arguments are as follows:
+.Bl -tag -width Ds
+.It Fl D Ar enc
+The default encoding.
+This is case-insensitive.
+See
+.Sx Algorithm
+and
+.Sx Encodings .
+.It Fl e Ar enc
+The document's encoding.
+This is case-insensitive.
+See
+.Sx Algorithm
+and
+.Sx Encodings .
+.It Ar file
+The input file.
+.El
+.Pp
+If
+.Ar file
+is not provided,
+.Nm
+accepts standard input.
+Output is written to standard output.
+Unicode characters in the ASCII range are printed as regular ASCII
+characters; those above this range are printed using the
+.Sq \e[uNNNN]
+format documented in
+.Xr mandoc_char 7 .
+.Pp
+If input bytes are improperly formed in the current encoding, they're
+passed unmodified to standard output.
+.Ss Encodings
+The
+.Nm
+utility accepts the
+.Ar utf\-8 ,
+.Ar us\-ascii ,
+and
+.Ar latin\-1
+encodings as arguments to
+.Fl D Ar enc
+or
+.Fl e Ar enc .
+.Ss Algorithm
+An encoding is chosen according to the following steps:
+.Bl -enum
+.It
+From the argument passed to
+.Fl e Ar enc .
+.It
+If a BOM exists, utf\-8 encoding is selected.
+.It
+From the argument passed to
+.Fl D Ar enc .
+.It
+If all else fails, Latin\-1 is used.
+.El
+.\" .Sh IMPLEMENTATION NOTES
+.\" Not used in OpenBSD.
+.\" .Sh RETURN VALUES
+.\" For sections 2, 3, & 9 only.
+.\" .Sh ENVIRONMENT
+.\" For sections 1, 6, 7, & 8 only.
+.\" .Sh FILES
+.Sh EXIT STATUS
+.Ex -std
+.\" .Sh EXAMPLES
+.\" .Sh DIAGNOSTICS
+.\" For sections 1, 4, 6, 7, & 8 only.
+.\" .Sh ERRORS
+.\" For sections 2, 3, & 9 only.
+.Sh SEE ALSO
+.Xr mandoc 1 ,
+.Xr mandoc_char 7
+.Sh STANDARDS
+The
+.Nm
+utility references the US-ASCII character set standard, ANSI_X3.4\-1968;
+the Latin\-1 character set standard, ISO/IEC 8859\-1:1998; the UTF\-8
+character set standard; and UCS (Unicode), ISO/IEC 10646.
+.Sh HISTORY
+The
+.Nm
+utility first appeared in the GNU troff
+.Pq Dq groff
+system in December 2005, authored by Tomohiro Kubota and Werner
+Lemberg.
+The implementation that is part of the
+.Xr mandoc 1
+utility appeared in May 2011.
+.Sh AUTHORS
+The
+.Nm
+utility was written by
+.An Kristaps Dzonsons Aq kristaps@bsd.lv .
+.\" .Sh CAVEATS
+.\" .Sh BUGS
+.\" .Sh SECURITY CONSIDERATIONS
+.\" Not used in OpenBSD.
diff --git a/preconv.c b/preconv.c
index b39578d9..4bd49607 100644
--- a/preconv.c
+++ b/preconv.c
@@ -85,7 +85,7 @@ static int
conv_latin_1(const struct buf *b)
{
size_t i;
- unsigned char c;
+ unsigned char cu;
const char *cp;
cp = b->buf + (int)b->offs;
@@ -97,8 +97,8 @@ conv_latin_1(const struct buf *b)
*/
for (i = b->offs; i < b->sz; i++) {
- c = (unsigned char)*cp++;
- c < 128 ? putchar(c) : printf("\\[u%.4X]", c);
+ cu = (unsigned char)*cp++;
+ cu < 128U ? putchar(cu) : printf("\\[u%.4X]", cu);
}
return(1);
@@ -120,6 +120,105 @@ conv_us_ascii(const struct buf *b)
static int
conv_utf_8(const struct buf *b)
{
+ int state, be;
+ unsigned int accum;
+ size_t i;
+ unsigned char cu;
+ const char *cp;
+ const long one = 1L;
+
+ cp = b->buf + (int)b->offs;
+ state = 0;
+ accum = 0U;
+ be = 0;
+
+ /* Quick test for big-endian value. */
+
+ if ( ! (*((char *)(&one))))
+ be = 1;
+
+ for (i = b->offs; i < b->sz; i++) {
+ cu = (unsigned char)*cp++;
+ if (state) {
+ if ( ! (cu & 128) || (cu & 64)) {
+ /* Bad sequence header. */
+ return(0);
+ }
+
+ /* Accept only legitimate bit patterns. */
+
+ if (cu > 191 || cu < 128) {
+ /* Bad in-sequence bits. */
+ return(0);
+ }
+
+ accum |= (cu & 63) << --state * 6;
+
+ /*
+ * Accum is held in little-endian order as
+ * stipulated by the UTF-8 sequence coding. We
+ * need to convert to a native big-endian if our
+ * architecture requires it.
+ */
+
+ if (0 == state && be)
+ accum = (accum >> 24) |
+ ((accum << 8) & 0x00FF0000) |
+ ((accum >> 8) & 0x0000FF00) |
+ (accum << 24);
+
+ if (0 == state) {
+ accum < 128U ? putchar(accum) :
+ printf("\\[u%.4X]", accum);
+ accum = 0U;
+ }
+ } else if (cu & (1 << 7)) {
+ /*
+ * Entering a UTF-8 state: if we encounter a
+ * UTF-8 bitmask, calculate the expected UTF-8
+ * state from it.
+ */
+ for (state = 0; state < 7; state++)
+ if ( ! (cu & (1 << (7 - state))))
+ break;
+
+ /* Accept only legitimate bit patterns. */
+
+ switch (state) {
+ case (4):
+ if (cu <= 244 && cu >= 240) {
+ accum = (cu & 7) << 18;
+ break;
+ }
+ /* Bad 4-sequence start bits. */
+ return(0);
+ case (3):
+ if (cu <= 239 && cu >= 224) {
+ accum = (cu & 15) << 12;
+ break;
+ }
+ /* Bad 3-sequence start bits. */
+ return(0);
+ case (2):
+ if (cu <= 223 && cu >= 194) {
+ accum = (cu & 31) << 6;
+ break;
+ }
+ /* Bad 2-sequence start bits. */
+ return(0);
+ default:
+ /* Bad sequence bit mask. */
+ return(0);
+ }
+ state--;
+ } else
+ putchar(cu);
+ }
+
+ if (0 != state) {
+ /* Bad trailing bits. */
+ return(0);
+ }
return(1);
}
@@ -211,9 +310,10 @@ int
main(int argc, char *argv[])
{
int i, ch, map, fd, rc;
- struct buf buf;
+ struct buf b;
const char *fn;
enum enc enc, def;
+ const char bom[3] = { 0xEF, 0xBB, 0xBF };
extern int optind;
extern char *optarg;
@@ -229,7 +329,7 @@ main(int argc, char *argv[])
enc = def = ENC__MAX;
map = 0;
- memset(&buf, 0, sizeof(struct buf));
+ memset(&b, 0, sizeof(struct buf));
while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
switch (ch) {
@@ -283,12 +383,16 @@ main(int argc, char *argv[])
}
}
- if ( ! read_whole_file(fn, fd, &buf, &map))
+ if ( ! read_whole_file(fn, fd, &b, &map))
goto out;
- if (ENC__MAX == enc) {
- /* TODO: search for BOM. */
- }
+ /* Try to read the UTF-8 BOM. */
+
+ if (ENC__MAX == enc)
+ if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
+ b.offs = 3;
+ enc = ENC_UTF_8;
+ }
/*
* No encoding has been detected.
@@ -299,15 +403,15 @@ main(int argc, char *argv[])
if (ENC__MAX == enc)
enc = ENC__MAX == def ? ENC_LATIN_1 : def;
- if ( ! (*encs[(int)enc].conv)(&buf))
+ if ( ! (*encs[(int)enc].conv)(&b))
goto out;
rc = EXIT_SUCCESS;
out:
if (map)
- munmap(buf.buf, buf.sz);
+ munmap(b.buf, b.sz);
else
- free(buf.buf);
+ free(b.buf);
if (fd > STDIN_FILENO)
close(fd);