Migrate `Bl -column' phrases to be MDOC_BODY instead of MDOC_HEAD. This

will make it easy for re-entrant parsing of `Ta' macros to fit in with
standard closure rules.

Added some more regressions for `Bl -column'.  Note that one should
fail, as documented in the TODO file.

Recorded change of AST BNF in mdoc.3.

10 files changed, 135 insertions, 64 deletions

diff --git a/TODO b/TODO
index 63ffd3dc..09871654 100644
--- a/TODO
+++ b/TODO
@@ -50,7 +50,11 @@
   noticed by espie@  Fri, 23 Apr 2010 17:10:35 +0200
   NEEDS MERGING TO bsd.lv
 
-
+- implement blank `Bl -column', such as
+  .Bl -column
+  .It foo Ta bar
+  .El
+  
 ************************************************************************
 * formatting issues: ugly output
 ************************************************************************
diff --git a/mdoc.3 b/mdoc.3
index 654013c5..ea7bedb4 100644
--- a/mdoc.3
+++ b/mdoc.3
@@ -221,9 +221,7 @@ where capitalised non-terminals represent nodes.
 .It mnode
 \(<- BLOCK | ELEMENT | TEXT
 .It BLOCK
-\(<- (HEAD [TEXT])+ [BODY [TEXT]] [TAIL [TEXT]]
-.It BLOCK
-\(<- BODY [TEXT] [TAIL [TEXT]]
+\(<- HEAD [TEXT] (BODY [TEXT])+ [TAIL [TEXT]]
 .It ELEMENT
 \(<- TEXT*
 .It HEAD
@@ -237,12 +235,14 @@ where capitalised non-terminals represent nodes.
 .El
 .Pp
 Of note are the TEXT nodes following the HEAD, BODY and TAIL nodes of
-the BLOCK production.
-These refer to punctuation marks.
+the BLOCK production: these refer to punctuation marks.
 Furthermore, although a TEXT node will generally have a non-zero-length
 string, in the specific case of
 .Sq \&.Bd \-literal ,
 an empty line will produce a zero-length string.
+Multiple body parts are only found in invocations of
+.Sq \&Bl \-column ,
+where a new body introduces a new phrase.
 .Sh EXAMPLES
 The following example reads lines from stdin and parses them, operating
 on the finished parse tree with
diff --git a/mdoc_html.c b/mdoc_html.c
index 22011403..c80d3c73 100644
--- a/mdoc_html.c
+++ b/mdoc_html.c
@@ -100,7 +100,8 @@ static	int		  mdoc_it_block_pre(MDOC_ARGS, enum mdoc_list,
 				int, struct roffsu *, struct roffsu *);
 static	int		  mdoc_it_head_pre(MDOC_ARGS, enum mdoc_list, 
 				struct roffsu *);
-static	int		  mdoc_it_body_pre(MDOC_ARGS, enum mdoc_list);
+static	int		  mdoc_it_body_pre(MDOC_ARGS, enum mdoc_list,
+				struct roffsu *);
 static	int		  mdoc_it_pre(MDOC_ARGS);
 static	int		  mdoc_lb_pre(MDOC_ARGS);
 static	int		  mdoc_li_pre(MDOC_ARGS);
@@ -877,7 +878,7 @@ mdoc_it_block_pre(MDOC_ARGS, enum mdoc_list type, int comp,
 
 /* ARGSUSED */
 static int
-mdoc_it_body_pre(MDOC_ARGS, enum mdoc_list type)
+mdoc_it_body_pre(MDOC_ARGS, enum mdoc_list type, struct roffsu *width)
 {
 	struct htmlpair	 tag;
 	struct roffsu	 su;
@@ -888,6 +889,12 @@ mdoc_it_body_pre(MDOC_ARGS, enum mdoc_list type)
 	case (LIST_ohang):
 		/* FALLTHROUGH */
 	case (LIST_column):
+		bufcat_su(h, "min-width", width);
+		bufcat_style(h, "clear", "none");
+		if (n->next)
+			bufcat_style(h, "float", "left");
+		PAIR_STYLE_INIT(&tag, h);
+		print_otag(h, TAG_DIV, 1, &tag);
 		break;
 	default:
 		/* 
@@ -920,12 +927,6 @@ mdoc_it_head_pre(MDOC_ARGS, enum mdoc_list type, struct roffsu *width)
 		print_otag(h, TAG_DIV, 0, &tag);
 		return(1);
 	case (LIST_column):
-		bufcat_su(h, "min-width", width);
-		bufcat_style(h, "clear", "none");
-		if (n->next && MDOC_HEAD == n->next->type)
-			bufcat_style(h, "float", "left");
-		PAIR_STYLE_INIT(&tag, h);
-		print_otag(h, TAG_DIV, 1, &tag);
 		break;
 	default:
 		bufcat_su(h, "min-width", width);
@@ -1048,25 +1049,21 @@ mdoc_it_pre(MDOC_ARGS)
 		break;
 	}
 
-	/* Flip to body/block processing. */
-
-	if (MDOC_BODY == n->type)
-		return(mdoc_it_body_pre(m, n, h, type));
-	if (MDOC_BLOCK == n->type)
-		return(mdoc_it_block_pre(m, n, h, type, comp,
-					&offs, &width));
-
-	/* Override column widths. */
-
-	if (LIST_column == type) {
+	if (LIST_column == type && MDOC_BODY == n->type) {
 		nn = n->parent->child;
-		for (i = 0; nn && nn != n; nn = nn->next, i++)
-			/* Counter... */ ;
+		for (i = 0; nn && nn != n; nn = nn->next)
+			if (MDOC_BODY == nn->type)
+				i++;
 		if (i < (int)bl->args->argv[wp].sz)
 			a2width(bl->args->argv[wp].value[i], &width);
 	}
 
-	return(mdoc_it_head_pre(m, n, h, type, &width));
+	if (MDOC_HEAD == n->type)
+		return(mdoc_it_head_pre(m, n, h, type, &width));
+	else if (MDOC_BODY == n->type)
+		return(mdoc_it_body_pre(m, n, h, type, &width));
+
+	return(mdoc_it_block_pre(m, n, h, type, comp, &offs, &width));
 }
 
 
diff --git a/mdoc_macro.c b/mdoc_macro.c
index e0e7ad6f..205512d6 100644
--- a/mdoc_macro.c
+++ b/mdoc_macro.c
@@ -937,6 +937,7 @@ blk_full(MACRO_PROT_ARGS)
 #ifdef	UGLY
 	struct mdoc_node *n;
 #endif
+	enum mdoc_type	  mtt;
 	enum mdoct	  ntok;
 	enum margserr	  ac, lac;
 	enum margverr	  av;
@@ -1030,15 +1031,9 @@ blk_full(MACRO_PROT_ARGS)
 			continue;
 		}
 
-		/* 
-		 * Open a head if one hasn't been opened. Re-open head
-		 * for phrases. 
-		 */
+		/* Open a head if one hasn't been opened. */
 
-		if (NULL == head || 
-				ARGS_PEND == ac ||
-				ARGS_PHRASE == ac || 
-				ARGS_PPHRASE == ac) {
+		if (NULL == head) {
 			if ( ! mdoc_head_alloc(m, line, ppos, tok))
 				return(0);
 			head = m->last;
@@ -1048,10 +1043,26 @@ blk_full(MACRO_PROT_ARGS)
 				ARGS_PEND == ac ||
 				ARGS_PPHRASE == ac) {
 			/*
+			 * If we haven't opened a body yet, rewind the
+			 * head; if we have, rewind that instead.
+			 */
+
+			mtt = body ? MDOC_BODY : MDOC_HEAD;
+			if ( ! rew_sub(mtt, m, tok, line, ppos))
+				return(0);
+			
+			/* Then allocate our body context. */
+
+			if ( ! mdoc_body_alloc(m, line, ppos, tok))
+				return(0);
+			body = m->last;
+
+			/*
 			 * Process phrases: set whether we're in a
 			 * partial-phrase (this effects line handling)
 			 * then call down into the phrase parser.
 			 */
+
 			if (ARGS_PPHRASE == ac)
 				m->flags |= MDOC_PPHRASE;
 			if (ARGS_PEND == ac && ARGS_PPHRASE == lac)
@@ -1064,11 +1075,6 @@ blk_full(MACRO_PROT_ARGS)
 				return(0);
 
 			m->flags &= ~MDOC_PPHRASE;
-
-			/* Close out active phrase. */
-
-			if ( ! rew_sub(MDOC_HEAD, m, tok, line, ppos))
-				return(0);
 			continue;
 		}
 
diff --git a/mdoc_term.c b/mdoc_term.c
index 943a880d..9bd89edc 100644
--- a/mdoc_term.c
+++ b/mdoc_term.c
@@ -700,7 +700,7 @@ termp_it_pre(DECL_ARGS)
 
 	switch (type) {
 	case (LIST_column):
-		if (MDOC_BODY == n->type)
+		if (MDOC_HEAD == n->type)
 			break;
 		/*
 		 * Imitate groff's column handling:
@@ -715,8 +715,13 @@ termp_it_pre(DECL_ARGS)
 		/* LINTED */
 		dcol = ncols < 5 ? 4 : ncols == 5 ? 3 : 1;
 
+		/*
+		 * Calculate the offset by applying all prior MDOC_BODY,
+		 * so we stop at the MDOC_HEAD (NULL == nn->prev).
+		 */
+
 		for (i = 0, nn = n->prev; 
-				nn && i < (int)ncols; 
+				nn->prev && i < (int)ncols; 
 				nn = nn->prev, i++)
 			offset += dcol + a2width
 				(&bl->args->argv[vals[2]], i);
@@ -869,15 +874,18 @@ termp_it_pre(DECL_ARGS)
 			p->flags |= TERMP_DANGLE;
 		break;
 	case (LIST_column):
-		if (MDOC_HEAD == n->type) {
-			assert(n->next);
-			if (MDOC_BODY == n->next->type)
-				p->flags &= ~TERMP_NOBREAK;
-			else
-				p->flags |= TERMP_NOBREAK;
-			if (n->prev) 
-				p->flags |= TERMP_NOLPAD;
-		}
+		if (MDOC_HEAD == n->type)
+			break;
+
+		if (NULL == n->next)
+			p->flags &= ~TERMP_NOBREAK;
+		else
+			p->flags |= TERMP_NOBREAK;
+
+		assert(n->prev);
+		if (MDOC_BODY == n->prev->type) 
+			p->flags |= TERMP_NOLPAD;
+
 		break;
 	case (LIST_diag):
 		if (MDOC_HEAD == n->type)
@@ -929,9 +937,9 @@ termp_it_pre(DECL_ARGS)
 		 * XXX - this behaviour is not documented: the
 		 * right-most column is filled to the right margin.
 		 */
-		if (MDOC_HEAD == n->type &&
-				MDOC_BODY == n->next->type &&
-				p->rmargin < p->maxrmargin)
+		if (MDOC_HEAD == n->type)
+			break;
+		if (NULL == n->next && p->rmargin < p->maxrmargin)
 			p->rmargin = p->maxrmargin;
 		break;
 	default:
@@ -985,7 +993,7 @@ termp_it_pre(DECL_ARGS)
 			return(0);
 		break;
 	case (LIST_column):
-		if (MDOC_BODY == n->type)
+		if (MDOC_HEAD == n->type)
 			return(0);
 		break;
 	default:
@@ -1017,7 +1025,7 @@ termp_it_post(DECL_ARGS)
 			term_newln(p);
 		break;
 	case (LIST_column):
-		if (MDOC_HEAD == n->type)
+		if (MDOC_BODY == n->type)
 			term_flushln(p);
 		break;
 	default:
diff --git a/mdoc_validate.c b/mdoc_validate.c
index ce90c0f3..8beae48a 100644
--- a/mdoc_validate.c
+++ b/mdoc_validate.c
@@ -928,6 +928,7 @@ post_an(POST_ARGS)
 static int
 post_it(POST_ARGS)
 {
+	/* FIXME: use mdoc_list! */
 	int		  type, i, cols;
 	struct mdoc_node *n, *c;
 
@@ -1017,15 +1018,16 @@ post_it(POST_ARGS)
 				return(0);
 		break;
 	case (MDOC_Column):
-		if (NULL == mdoc->last->head->child)
-			if ( ! mdoc_nmsg(mdoc, mdoc->last, MANDOCERR_NOARGS))
-				return(0);
-		if (mdoc->last->body->child)
-			if ( ! mdoc_nmsg(mdoc, mdoc->last, MANDOCERR_BODYLOST))
+		assert(NULL == mdoc->last->head->child);
+		if (NULL == mdoc->last->body->child)
+			if ( ! mdoc_nmsg(mdoc, mdoc->last, MANDOCERR_NOBODY))
 				return(0);
+
+		/* Count up the number of columns.  */
 		c = mdoc->last->child;
-		for (i = 0; c && MDOC_HEAD == c->type; c = c->next)
-			i++;
+		for (i = 0; c; c = c->next)
+			if (MDOC_BODY == c->type)
+				i++;
 
 		if (i < cols) {
 			if ( ! mdoc_vmsg(mdoc, MANDOCERR_ARGCOUNT,
diff --git a/regress/mdoc/It/nested-punctuation.in b/regress/mdoc/It/nested-punctuation.in
new file mode 100644
index 00000000..998d5ee2
--- /dev/null
+++ b/regress/mdoc/It/nested-punctuation.in
@@ -0,0 +1,10 @@
+.Dd $Mdocdate$
+.Dt FOO
+.Os
+.Sh NAME
+.Nm foo
+.Nd bar
+.Sh DESCRIPTION
+.Bl -column asdfasdf asdfasdf 
+.It ( asdf ) ; Ta ( ( asdf :
+.El
diff --git a/regress/mdoc/It/nocolspec.in b/regress/mdoc/It/nocolspec.in
new file mode 100644
index 00000000..958bd9bd
--- /dev/null
+++ b/regress/mdoc/It/nocolspec.in
@@ -0,0 +1,10 @@
+.Dd $Mdocdate$
+.Dt FOO
+.Os
+.Sh NAME
+.Nm foo
+.Nd bar
+.Sh DESCRIPTION
+.Bl -column
+.It asdf Ta asdf
+.El
diff --git a/regress/mdoc/It/pre-punct.in b/regress/mdoc/It/pre-punct.in
new file mode 100644
index 00000000..21be7d83
--- /dev/null
+++ b/regress/mdoc/It/pre-punct.in
@@ -0,0 +1,17 @@
+.Dd $Mdocdate$
+.Dt FOO
+.Os
+.Sh NAME
+.Nm foo
+.Nd bar
+.Sh DESCRIPTION
+.Bl -column asdfasdf asdfasdf asdfasdf
+.It ( asdf	asdf	asdf
+.El
+.Bl -column asdfasdf asdfasdf asdfasdf
+.It ( asdf ; Ta ( asdf ; Ta asdf
+.El
+.Bl -tag -width Ds
+.It asdf
+asdf
+.El
diff --git a/regress/mdoc/It/tab-macros.in b/regress/mdoc/It/tab-macros.in
new file mode 100644
index 00000000..0c7b367e
--- /dev/null
+++ b/regress/mdoc/It/tab-macros.in
@@ -0,0 +1,17 @@
+.Dd $Mdocdate$
+.Dt FOO
+.Os
+.Sh NAME
+.Nm foo
+.Nd bar
+.Sh DESCRIPTION
+.Bl -column asdfasdf asdfasdf asdfasdf
+.It Fl asdf	asdf	asdf
+.El
+.Bl -column asdfasdf asdfasdf asdfasdf
+.It Ar asdf ; Ta Fl asdf ; Ta asdf
+.El
+.Bl -tag -width Ds
+.It asdf
+asdf
+.El