diff options
| -rw-r--r-- | Makefile | 16 | ||||
| -rw-r--r-- | mandoc.db.5 | 144 | ||||
| -rw-r--r-- | mansearch.3 | 227 |
3 files changed, 383 insertions, 4 deletions
@@ -1,7 +1,7 @@ .PHONY: clean install installcgi installwww .SUFFIXES: .md5 .h .h.html -.SUFFIXES: .1 .3 .7 .8 -.SUFFIXES: .1.html .3.html .7.html .8.html +.SUFFIXES: .1 .3 .5 .7 .8 +.SUFFIXES: .1.html .3.html .5.html .7.html .8.html VERSION = 1.13.0 @@ -123,6 +123,7 @@ SRCS = Makefile \ mandoc.1 \ mandoc.3 \ mandoc.c \ + mandoc.db.5 \ mandoc.h \ mandoc_aux.c \ mandoc_aux.h \ @@ -132,6 +133,7 @@ SRCS = Makefile \ manpage.c \ manpath.c \ manpath.h \ + mansearch.3 \ mansearch.c \ mansearch.h \ mansearch_const.c \ @@ -278,7 +280,9 @@ WWW_MANS = apropos.1.html \ mandoc.1.html \ preconv.1.html \ mandoc.3.html \ + mansearch.3.html \ tbl.3.html \ + mandoc.db.5.html \ eqn.7.html \ man.7.html \ mandoc_char.7.html \ @@ -288,6 +292,8 @@ WWW_MANS = apropos.1.html \ mandocdb.8.html \ man.h.html \ mandoc.h.html \ + manpath.h.html \ + mansearch.h.html \ mdoc.h.html $(WWW_MANS): mandoc @@ -314,12 +320,14 @@ install: all mkdir -p $(DESTDIR)$(INCLUDEDIR) mkdir -p $(DESTDIR)$(MANDIR)/man1 mkdir -p $(DESTDIR)$(MANDIR)/man3 + mkdir -p $(DESTDIR)$(MANDIR)/man5 mkdir -p $(DESTDIR)$(MANDIR)/man7 $(INSTALL_PROGRAM) $(ALLBIN) $(DESTDIR)$(BINDIR) $(INSTALL_LIB) libmandoc.a $(DESTDIR)$(LIBDIR) $(INSTALL_LIB) man.h mdoc.h mandoc.h $(DESTDIR)$(INCLUDEDIR) $(INSTALL_MAN) mandoc.1 preconv.1 demandoc.1 $(DESTDIR)$(MANDIR)/man1 - $(INSTALL_MAN) mandoc.3 tbl.3 $(DESTDIR)$(MANDIR)/man3 + $(INSTALL_MAN) mandoc.3 mansearch.3 tbl.3 $(DESTDIR)$(MANDIR)/man3 + $(INSTALL_MAN) mandoc.db.5 $(DESTDIR)$(MANDIR)/man5 $(INSTALL_MAN) man.7 mdoc.7 roff.7 eqn.7 tbl.7 mandoc_char.7 $(DESTDIR)$(MANDIR)/man7 $(INSTALL_DATA) example.style.css $(DESTDIR)$(EXAMPLEDIR) @@ -377,5 +385,5 @@ config.h: configure config.h.pre config.h.post $(TESTSRCS) .h.h.html: highlight -I $< >$@ -.1.1.html .3.3.html .7.7.html .8.8.html: +.1.1.html .3.3.html .5.5.html .7.7.html .8.8.html: ./mandoc -Thtml -Wall,stop -Ostyle=style.css,man=%N.%S.html,includes=%I.html $< >$@ diff --git a/mandoc.db.5 b/mandoc.db.5 new file mode 100644 index 00000000..0cae2fd0 --- /dev/null +++ b/mandoc.db.5 @@ -0,0 +1,144 @@ +.\" $Id$ +.\" +.\" Copyright (c) 2014 Ingo Schwarze <schwarze@openbsd.org> +.\" +.\" Permission to use, copy, modify, and distribute this software for any +.\" purpose with or without fee is hereby granted, provided that the above +.\" copyright notice and this permission notice appear in all copies. +.\" +.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +.\" +.Dd $Mdocdate$ +.Dt MANDOC.DB 5 +.Os +.Sh NAME +.Nm mandoc.db +.Nd manual page database +.Sh DESCRIPTION +The +.Nm +SQLite3 file format is used to store information about installed manual +pages to facilitate semantic searching for manuals. +Each manual page tree contains its own +.Nm +file; see +.Sx FILES +for examples. +.Pp +Such database files are generated by +.Xr makewhatis 8 +and used by +.Xr apropos 1 +and +.Xr whatis 1 . +.Pp +One line in the following tables describes: +.Bl -tag -width Ds +.It Sy mpages +One physical manual page file, no matter how many times and under which +names it may appear in the file system. +.It Sy mlinks +One entry in the file system, no matter which content it points to. +.It Sy names +One manual page name, no matter whether it appears in a page header, +in a NAME or SYNOPSIS section, or as a file name. +.It Sy keys +One chunk of text from some macro invocation. +.El +.Pp +Each record in the latter three tables uses its +.Va pageid +column to point to a record in the +.Sy mpages +table. +.Pp +The other columns are as follows; unless stated otherwise, they are +of type +.Vt TEXT . +.Bl -tag -width mpages.desc +.It Sy mpages.desc +The description line +.Pq Sq \&Nd +of the page. +.It Sy mpages.form +The +.Vt INTEGER +1 if the page is unformatted, i.e. in +.Xr mdoc 7 +or +.Xr man 7 +format, and 2 if it is formatted, i.e. a +.Sq cat +page. +.It Sy mlinks.sec +The manual section as found in the subdirectory name. +.It Sy mlinks.arch +The manual architecture as found in the subdirectory name, or +.Qq any . +.It Sy mlinks.name +The manual name as found in the file name. +.It Sy names.bits +An +.Vt INTEGER +bit mask telling whether the name came from a header line, from the +NAME or SYNOPSIS section, or from a file name. +Bits are defined in +.In mansearch.h . +.It Sy names.name +The name itself. +.It Sy keys.bits +An +.Vt INTEGER +bit mask telling which semantic contexts the key was found in; +defined in +.In mansearch.h , +documented in +.Xr apropos 1 . +.It Sy keys.key +The string found in those contexts. +.El +.Sh FILES +.Bl -tag -width /usr/share/mandoc.db -compact +.It Pa /usr/share/mandoc.db +The manual page database for the base system. +.It Pa /usr/X11R6/mandoc.db +The same for the +.Xr X 7 +Window System. +.It Pa /usr/local/mandoc.db +The same for +.Xr packages 7 . +.El +.Sh SEE ALSO +.Xr apropos 1 , +.Xr man 1 , +.Xr sqlite3 1 , +.Xr whatis 1 , +.Xr mansearch 3 , +.Xr makewhatis 8 +.Sh HISTORY +A manual page database +.Pa /usr/lib/whatis +first appeared in +.Bx 2 . +The present format first appeared in +.Ox 5.6 . +.Sh AUTHORS +.An -nosplit +The original version of +.Xr makewhatis 8 +was written by +.An Bill Joy +in 1979. +An SQLite3 version was first implemented by +.An Kristaps Dzonsons Aq Mt kristaps@bsd.lv +in 2012. +The present database format was designed by +.An Ingo Schwarze Aq Mt schwarze@openbsd.org +in 2014. diff --git a/mansearch.3 b/mansearch.3 new file mode 100644 index 00000000..bb382db4 --- /dev/null +++ b/mansearch.3 @@ -0,0 +1,227 @@ +.\" $Id$ +.\" +.\" Copyright (c) 2014 Ingo Schwarze <schwarze@openbsd.org> +.\" +.\" Permission to use, copy, modify, and distribute this software for any +.\" purpose with or without fee is hereby granted, provided that the above +.\" copyright notice and this permission notice appear in all copies. +.\" +.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +.\" +.Dd $Mdocdate$ +.Dt MANSEARCH 3 +.Os +.Sh NAME +.Nm mansearch , +.Nm mansearch_setup +.Nd search manual page databases +.Sh SYNOPSIS +.In manpath.h +.In mansearch.h +.Ft int +.Fo mansearch_setup +.Fa "int start" +.Fc +.Ft int +.Fo mansearch +.Fa "const struct mansearch *search" +.Fa "const struct manpaths *paths" +.Fa "int argc" +.Fa "char *argv[]" +.Fa "const char *outkey" +.Fa "struct manpage **res" +.Fa "size_t *sz" +.Fc +.Sh DESCRIPTION +The +.Fn mansearch +function returns information about manuals matching a search query from a +.Xr mandoc.db 5 +SQLite3 database. +.Pp +The query arguments are as follows: +.Bl -tag -width Ds +.It Fa "const struct mansearch *search" +Search options, defined in +.In mansearch.h . +.It Fa "const struct manpaths *paths" +Directories to be searched, defined in +.In manpath.h . +.It Fa "int argc" , "char *argv[]" +Search criteria, usually taken from the command line. +.El +.Pp +The +.Fa "const char *outkey" +selects which data to return in the +.Va output +field of the +.Fa res +structures. +It takes any of the macro keys defined in +.Pa mansearch_const.c +and described in +.Xr apropos 1 . +.Pp +The output arguments are as follows: +.Bl -tag -width Ds +.It Fa "struct manpage **res" +Returns a pointer to an array of result structures defined in +.In mansearch.h . +The user is expected to call +.Xr free 3 +on the +.Va file , +.Va names , +and +.Va output +fields of all structures, as well as the +.Fa res +array itself. +.It Fa "size_t *sz" +Returns the number of result structures contained in +.Fa res . +.El +.Pp +To speed up searches, the +.Fn mansearch_setup +function can optionally be called with a +.Fa start +argument of 1 before +.Fn mansearch +to set up an SQLite3 pagecache. +If it was called, it has to be called again with a +.Fa start +argument of 0 after the last call to +.Fn mansearch +to release the memory used for the pagecache. +.Sh IMPLEMENTATION NOTES +For each manual page tree, the search is done in two steps. +In the first step, a list of pages matching the search criteria is built. +In the second step, the requested information about these pages is +retrieved from the database and assembled into the +.Fa res +array. +.Pp +All function mentioned here are defined in the file +.Pa mansearch.c . +No functions except +.Fn mansearch +and +.Fn sql_statement +build any SQL code, and no functions except +.Fn mansearch , +.Fn buildnames , +and +.Fn buildoutput +execute it. +.Ss Finding matches +The query is built using the following grammar: +.Bd -literal -offset indent +<query> ::= "SELECT * FROM mpages WHERE" <condition> +<condition> ::= "(" <condition> ")" | + <condition> "OR" <condition> | + <condition> "AND" <condition> | + "desc" <operator> "?" | + "id IN (SELECT pageid FROM" <subquery> ")" +<subquery> ::= "names WHERE name" <operator> "?" | + "keys WHERE key" <operator> "? AND bits & ?" +<operator> ::= "MATCH" | "REGEXP" +.Ed +.Pp +The MATCH and REGEXP operators are implemented by the functions +.Fn sql_match +and +.Fn sql_regexp , +respectively. +This is required because SQLite3 natively neither supports +case-insensitive substring matching nor regular expression matching, +but only string identity, shell globbing, and the weird home-brewed +LIKE operator. +.Pp +Command line parsing is done by the function +.Fn exprcomp +building a singly linked list of +.Vt expr +structures, using the helper functions +.Fn exprterm +and +.Fn exprspec . +The resulting SQL statement is assembled by the function +.Fn sql_statement +and evaluated in the main loop of the +.Fn mansearch +function. +.Ss Assembling the results +The names, sections, and architectures of the manuals found +are assembled into the +.Va names +field of the result structure by the function +.Fn buildnames , +using the following query: +.Pp +.Dl "SELECT * FROM mlinks WHERE pageid=? ORDER BY sec, arch, name" +.Pp +If the +.Fa outkey +differs from +.Qq Nd , +the requested output data is assembled into the +.Va output +field of the result structure by the function +.Fn buildoutput , +using the following query: +.Pp +.Dl "SELECT * FROM keys WHERE pageid=? AND bits & ?" +.Sh FILES +.Bl -tag -width mandoc.db -compact +.It Pa mandoc.db +The manual page database. +.El +.Sh EXAMPLES +The simplest invocation +.Pp +.Dl apropos keyword +.Pp +results in the following SQL query: +.Bd -literal +SELECT * FROM mpages WHERE ( + id IN (SELECT pageid FROM names WHERE name MATCH 'keyword') OR + desc MATCH 'keyword' +); +.Ed +.Pp +A more complicated request like +.Pp +.Dl apropos -s 2 Nm,Xr=getuid +.Pp +results in: +.Bd -literal +SELECT * FROM mpages WHERE ( + id IN (SELECT pageid FROM names WHERE name MATCH 'getuid') OR + id IN (SELECT pageid FROM keys WHERE key MATCH 'getuid' AND bits & 4) +) AND id IN (SELECT pageid FROM keys WHERE key REGEXP '^2$' AND bits & 2); +.Ed +.Sh SEE ALSO +.Xr apropos 1 , +.Xr mandoc.db 5 , +.Xr makewhatis 8 +.Sh HISTORY +The +.Fn mansearch +subsystem first appeared in +.Ox 5.6 . +.Sh AUTHORS +.An -nosplit +A module to search manual page databases was first written by +.An Kristaps Dzonsons Aq Mt kristaps@bsd.lv +in 2011, at first using the Berkeley DB; +he rewrote it for SQLite3 in 2012. +The current version received major changes from +.An Ingo Schwarze Aq Mt schwarze@openbsd.org . |