Merge branch 'syntax_aware' into master

2 files changed, 264 insertions, 31 deletions

diff --git a/Readme.md b/Readme.md
index 87f27ec..31a06fd 100644
--- a/Readme.md
+++ b/Readme.md
@@ -2,19 +2,20 @@
 
 A spellchecking lua plugin for the [vis editor](https://github.com/martanne/vis).
 
-## installation
+## Installation
 
 1. Download `spellcheck.lua` or clone this repository
 2. Load the plugin in your `visrc.lua` with `require(path/to/plugin/spellcheck)`
 
-## usage
+## Usage
 
 + To enable highlighting of misspelled words press `<Ctrl-w>e` in normal mode.
 + To disable highlighting press `<Ctrl-w>d` in normal mode.
++ To toggle highlighting press `<F7>` in normal mode.
 + To correct the word under the cursor press `<Ctrl+w>w` in normal mode.
 + To ignore the word under the cursor press `<Ctrl+w>i` in normal mode.
 
-## configuration
+## Configuration
 
 The module table returned from `require(...)` has some configuration options:
 
@@ -24,8 +25,12 @@ The module table returned from `require(...)` has some configuration options:
 	* default: `enchant -l -d %s` 
 * `lang`: The name of the used dictionary. `lang` is inserted in the cmd-strings at `%s`.
 	* default: `$LANG` or `en_US`
-* `typo_style`: The style string with which misspellings should be highlighted
+* `typo_style`: The style string with which misspellings should be highlighted when using the _full viewport_ method
 	* default: `fore:red`
+* `check_tokens`: A table mapping all token names we consider for spellchecking to true
+	* default: `{[vis.lexers.STRING]=true, [vis.lexers.COMMENT]=true, [vis.lexers.DEFAULT]=true}`
+* `disable_syntax_awareness`: Disable the syntax aware spellchecking and use always _full viewport_
+	* default: `false`
 
 A possible configuration could look like this:
 
diff --git a/spellcheck.lua b/spellcheck.lua
index 9d8a6ed..90ba907 100644
--- a/spellcheck.lua
+++ b/spellcheck.lua
@@ -3,8 +3,10 @@
 
 local spellcheck = {}
 spellcheck.lang = os.getenv("LANG"):sub(0,5) or "en_US"
-local supress_output = ">/dev/null 2>/dev/null"
-if os.execute("type enchant "..supress_output) then
+local supress_stdout = " >/dev/null"
+local supress_stderr = " 2>/dev/null"
+local supress_output = supress_stdout .. supress_stderr
+if os.execute("type enchant"..supress_output) then
 	spellcheck.cmd = "enchant -d %s -a"
 	spellcheck.list_cmd = "enchant -l -d %s"
 elseif os.execute("type enchant-2"..supress_output) then
@@ -21,14 +23,131 @@ else
 end
 
 spellcheck.typo_style = "fore:red"
-spellcheck.enabled = {}
+spellcheck.check_full_viewport = {}
+spellcheck.disable_syntax_awareness = false
 
+spellcheck.check_tokens = {
+	[vis.lexers.STRING] = true,
+	[vis.lexers.COMMENT] = true,
+	[vis.lexers.DEFAULT] = true,
+}
+
+-- Return nil or a string of misspelled word in a specific file range or text
+-- by calling the spellchecker's list command.
+-- If given a range we will use vis:pipe to get our typos from the spellchecker.
+-- If a string was passed we call the spellchecker ourself and redirect its stdout
+-- to a temporary file. See http://lua-users.org/lists/lua-l/2007-10/msg00189.html.
+-- The returned string consists of each misspell followed by a newline.
+local function get_typos(range_or_text)
+	local cmd = spellcheck.list_cmd:format(spellcheck.lang)
+	local typos = nil
+	if type(range_or_text) == "string" then
+		local text = range_or_text
+		local tmp_name = os.tmpname()
+		local full_cmd = cmd .. "> " .. tmp_name .. supress_stderr
+		local proc = assert(io.popen(full_cmd, "w"))
+		proc:write(text)
+		-- this error detection may need lua5.2
+		local success, reason, exit_code = proc:close()
+		if not success then
+			vis:info("calling " .. cmd .. " failed ("..exit_code..")")
+			return nil
+		end
+
+		local tmp_file = assert(io.open(tmp_name, "r"))
+		typos = tmp_file:read("*a")
+		tmp_file:close()
+		os.remove(tmp_name)
+	else
+		local range = range_or_text
+		local ret, so, se = vis:pipe(vis.win.file, range, cmd)
+
+		if ret ~= 0 then
+			vis:info("calling " .. cmd .. " failed ("..ret..")")
+			return nil
+		end
+		typos = so
+	end
+
+	return typos
+end
+
+-- plugin global list of ignored typos
 local ignored = {}
 
-local last_viewport, last_typos = nil, ""
+-- Return an iterator over all not ignored typos and their positions in text.
+-- The returned iterator is a self contained statefull iterator function closure.
+-- Which will return the next typo and its start and finish in the text, starting by 1.
+local function typo_iter(text, typos, ignored)
+	local index = 1
+	local unfiltered_iterator, iter_state = typos:gmatch("(.-)\n")
+
+-- see https://stackoverflow.com/questions/6705872/how-to-escape-a-variable-in-lua
+	local escape_lua_pattern
+	do
+		local matches =
+			{
+				["^"] = "%^";
+				["$"] = "%$";
+				["("] = "%(";
+				[")"] = "%)";
+				["%"] = "%%";
+				["."] = "%.";
+				["["] = "%[";
+				["]"] = "%]";
+				["*"] = "%*";
+				["+"] = "%+";
+				["-"] = "%-";
+				["?"] = "%?";
+			}
+
+		escape_lua_pattern = function(s)
+			return (s:gsub(".", matches))
+		end
+	end
+
+	return function(foo, bar)
+		repeat
+			typo = unfiltered_iterator(iter_state)
+		until(not typo or (typo ~= "" and not ignored[typo]))
+
+		if typo then
+			-- to prevent typos from being found in correct words before them
+			-- ("stuff stuf", "broken ok", ...)
+			-- we match typos only when they are enclosed in non-letter characters.
+			local start, finish = text:find("[%A]" .. escape_lua_pattern(typo) .. "[%A]", index)
+			-- typo was not found by our pattern this means it must be either
+			-- the first or last word in the text
+			if not start then
+				-- check start of text
+				start = 1
+				finish = #typo
+				-- typo is not the beginning must be the end of text
+				if text:sub(start, finish) ~= typo then
+					start = #text - #typo + 1
+					finish = start + #typo - 1
+				end
+
+				if text:sub(start, finish) ~= typo then
+					vis:info(string.format("can't find typo %s after %d. Please report this bug.",
+										   typo, index))
+				end
+			-- our pettern [%A]typo[%A] found it
+			else
+				start = start + 1 -- ignore leading non letter char
+				finish = finish - 1 -- ignore trailing non letter char
+			end
+			index = finish
+
+			return typo, start, finish
+		end
+	end
+end
+
+local last_viewport, last_data, last_typos = nil, "", ""
 
 vis.events.subscribe(vis.events.WIN_HIGHLIGHT, function(win)
-	if not spellcheck.enabled[win] or not win:style_define(42, spellcheck.typo_style) then
+	if not spellcheck.check_full_viewport[win] or not win:style_define(42, spellcheck.typo_style) then
 		return false
 	end
 	local viewport = win.viewport
@@ -39,23 +158,14 @@ vis.events.subscribe(vis.events.WIN_HIGHLIGHT, function(win)
 	if last_viewport == viewport_text then
 		typos = last_typos
 	else
-		local cmd = spellcheck.list_cmd:format(spellcheck.lang)
-		local ret, so, se = vis:pipe(win.file, viewport, cmd)
-		if ret ~= 0 then
-			vis:message("calling " .. cmd .. " failed ("..se..")")
+		typos = get_typos(viewport) or ""
+		if not typos then
 			return false
 		end
-		typos = so or ""
 	end
 
-	local corrections_iter = typos:gmatch("(.-)\n")
-	local index = 1
-	for typo in corrections_iter do
-		if not ignored[typo] then
-			local start, finish = viewport_text:find(typo, index, true)
-			win:style(42, viewport.start + start - 1, viewport.start + finish)
-			index = finish
-		end
+	for typo, start, finish in typo_iter(viewport_text, typos, ignored) do
+		win:style(42, viewport.start + start - 1, viewport.start + finish)
 	end
 
 	last_viewport = viewport_text
@@ -63,26 +173,142 @@ vis.events.subscribe(vis.events.WIN_HIGHLIGHT, function(win)
 	return true
 end)
 
+local wrapped_lex_funcs = {}
+
+local wrap_lex_func = function(old_lex_func)
+	local old_new_tokens = {}
+
+	return function(lexer, data, index, redrawtime_max)
+		local tokens, timedout = old_lex_func(lexer, data, index, redrawtime_max)
+
+		-- quit early if the lexer already took to long
+		-- TODO: investigate further if timedout is actually set by the lexer.
+		--       As I understand lpeg.match used by lexer.lex timedout will always be nil
+		if timeout then
+			return tokens, timedout
+		end
+
+		local new_tokens = {}
+
+		local typos = ""
+		if last_data ~= data
+		then
+			typos = get_typos(data)
+			if not typos then
+				return tokens, timedout
+			end
+			last_data = data
+		else
+			return old_new_tokens
+		end
+
+		local i = 1
+		for typo, typo_start, typo_end in typo_iter(data, typos, ignored) do
+			repeat
+				-- no tokens left
+				if i > #tokens -1 then
+					break
+				end
+
+				local token_type = tokens[i]
+				local token_start = (tokens[i-1] or 1) - 1
+				local token_end = tokens[i+1]
+
+				-- the current token ends before our typo -> append to new stream
+				-- or is not spellchecked
+				if token_end < typo_start or not spellcheck.check_tokens[token_type] then
+					table.insert(new_tokens, token_type)
+					table.insert(new_tokens, token_end)
+
+					-- done with this token -> advance token stream
+					i = i + 2
+				-- typo and checked token overlap
+				else
+					local pre_typo_end = typo_start - 1
+					-- unchanged token part before typo
+					if pre_typo_end > token_start then
+						table.insert(new_tokens, token_type)
+						table.insert(new_tokens, pre_typo_end + 1)
+					end
+
+					-- highlight typo
+					table.insert(new_tokens, vis.lexers.ERROR)
+					-- the typo spans multiple tokens
+					if token_end < typo_end then
+						table.insert(new_tokens, token_end + 1)
+						i = i + 2
+					else
+						table.insert(new_tokens, typo_end + 1)
+					end
+				end
+			until(not token_end or token_end >= typo_end)
+		end
+
+		-- add tokens left after we handled all typos
+		for i = i, #tokens, 1 do
+			table.insert(new_tokens, tokens[i])
+		end
+
+		old_new_tokens = new_tokens
+		return new_tokens, timedout
+	end
+end
+
+local enable_spellcheck = function()
+	-- prevent wrapping the lex function multiple times
+	if wrapped_lex_funcs[vis.win] then
+		return
+	end
+
+	if not spellcheck.disable_syntax_awareness and vis.win.syntax and vis.lexers.load then
+		local lexer = vis.lexers.load(vis.win.syntax, nil, true)
+		if lexer and lexer.lex then
+			local old_lex_func = lexer.lex
+			wrapped_lex_funcs[vis.win] = old_lex_func
+			lexer.lex = wrap_lex_func(old_lex_func)
+			-- reset last data to enforce new highlighting
+			last_data = ""
+			return
+		end
+	end
+
+	-- fallback check spellcheck the full viewport
+	spellcheck.check_full_viewport[vis.win] = true
+end
+
+local is_spellcheck_enabled = function()
+	return spellcheck.check_full_viewport[vis.win] or wrapped_lex_funcs[vis.win]
+end
+
 vis:map(vis.modes.NORMAL, "<C-w>e", function(keys)
-	spellcheck.enabled[vis.win] = true
-	return 0
+	enable_spellcheck()
 end, "Enable spellchecking in the current window")
 
+local disable_spellcheck = function()
+	local old_lex_func = wrapped_lex_funcs[vis.win]
+	if old_lex_func then
+		local lexer = vis.lexers.load(vis.win.syntax, nil, true)
+		lexer.lex = old_lex_func
+		wrapped_lex_funcs[vis.win] = nil
+	else
+		spellcheck.check_full_viewport[vis.win] = nil
+	end
+end
+
 vis:map(vis.modes.NORMAL, "<C-w>d", function(keys)
-	spellcheck.enabled[vis.win] = nil
+	disable_spellcheck()
 	-- force new highlight
 	vis.win:draw()
-	return 0
 end, "Disable spellchecking in the current window")
 
 -- toggle spellchecking on <F7>
 -- <F7> is used by some word processors (LibreOffice) for spellchecking
 -- Thanks to @leorosa for the hint.
 vis:map(vis.modes.NORMAL, "<F7>", function(keys)
-	if not spellcheck.enabled[vis.win] then
-		spellcheck.enabled[vis.win] = true
+	if not is_spellcheck_enabled() then
+		enable_spellcheck()
 	else
-		spellcheck.enabled[vis.win] = nil
+		disable_spellcheck()
 		vis.win:draw()
 	end
 	return 0
@@ -122,7 +348,7 @@ vis:map(vis.modes.NORMAL, "<C-w>w", function(keys)
 
 	-- select a correction
 	local cmd = 'printf "' .. suggestions:gsub(", ", "\\n") .. '\\n" | vis-menu'
-	local f = io.popen(cmd)
+	local f = assert(io.popen(cmd))
 	local correction = f:read("*all")
 	f:close()
 	-- trim correction
@@ -157,8 +383,10 @@ end, "Ignore misspelled word")
 vis:option_register("spelllang", "string", function(value, toggle)
 	spellcheck.lang = value
 	vis:info("Spellchecking language is now "..value)
-	-- force new highlight
+	-- force new highlight for full viewport
 	last_viewport = nil
+	-- force new highlight for syntax aware
+	last_data = nil
 	return true
 end, "The language used for spellchecking")