path: root/src/epy_reader/parser.py

import curses
import dataclasses
import re
import textwrap
from html import unescape
from html.parser import HTMLParser
from typing import Dict, List, Mapping, Optional, Sequence, Set, Tuple, Union
from urllib.parse import unquote

from epy_reader.models import CharPos, InlineStyle, TextMark, TextSpan, TextStructure


class HTMLtoLines(HTMLParser):
    para = {"p", "div"}
    inde = {"q", "dt", "dd", "blockquote"}
    pref = {"pre"}
    bull = {"li"}
    hide = {"script", "style", "head"}
    ital = {"i", "em"}
    bold = {"b", "strong"}
    # hide = {"script", "style", "head", ", "sub}
    # sup_lookup = "⁰¹²³⁴⁵⁶⁷⁸⁹"
    # sub_lookup = "₀₁₂₃₄₅₆₇₈₉"

    attr_bold = curses.A_BOLD
    try:
        attr_italic = curses.A_ITALIC
    except AttributeError:
        try:
            attr_italic = curses.A_UNDERLINE
        except AttributeError:
            attr_italic = curses.A_NORMAL

    @staticmethod
    def _mark_to_spans(text: Sequence[str], marks: Sequence[TextMark]) -> List[TextSpan]:
        """
        Convert text marks in line of text to per line text span.
        Keeping duplicate spans.
        """
        spans: List[TextSpan] = []
        for mark in marks:
            if mark.is_valid():
                # mypy issue, should be handled by mark.is_valid()
                assert mark.end is not None
                if mark.start.row == mark.end.row:
                    spans.append(
                        TextSpan(start=mark.start, n_letters=mark.end.col - mark.start.col)
                    )
                else:
                    spans.append(
                        TextSpan(
                            start=mark.start, n_letters=len(text[mark.start.row]) - mark.start.col
                        )
                    )
                    for nth_line in range(mark.start.row + 1, mark.end.row):
                        spans.append(
                            TextSpan(
                                start=CharPos(row=nth_line, col=0), n_letters=len(text[nth_line])
                            )
                        )
                    spans.append(
                        TextSpan(start=CharPos(row=mark.end.row, col=0), n_letters=mark.end.col)
                    )

        return spans  # list(set(spans))

    @staticmethod
    def _adjust_wrapped_spans(
        wrapped_lines: Sequence[str],
        span: TextSpan,
        *,
        line_adjustment: int = 0,
        left_adjustment: int = 0,
    ) -> List[TextSpan]:
        """
        Adjust text span to wrapped lines.
        Not perfect, but should be good enough considering
        the limitation on commandline interface.
        """

        # current_row = span.start.row + line_adjustment
        current_row = line_adjustment
        start_col = span.start.col
        end_col = start_col + span.n_letters

        prev = 0  # chars length before current line
        spans: List[TextSpan] = []
        for n, line in enumerate(wrapped_lines):
            # + 1 compensates textwrap.wrap(*args, replace_whitespace=True, drop_whitespace=True)
            line_len = len(line) + 1
            current = prev + line_len  # chars length before next line

            # -:unmarked *:marked
            # |------*****--------|
            if start_col in range(prev, current) and end_col in range(prev, current):
                spans.append(
                    TextSpan(
                        start=CharPos(row=current_row + n, col=start_col - prev + left_adjustment),
                        n_letters=span.n_letters,
                    )
                )

            # |----------*********|
            elif start_col in range(prev, current):
                spans.append(
                    TextSpan(
                        start=CharPos(row=current_row + n, col=start_col - prev + left_adjustment),
                        n_letters=current - start_col - 1,  # -1: dropped whitespace
                    )
                )

            # |********-----------|
            elif end_col in range(prev, current):
                spans.append(
                    TextSpan(
                        start=CharPos(row=current_row + n, col=0 + left_adjustment),
                        n_letters=end_col - prev + 1,  # +1: dropped whitespace
                    )
                )

            # |*******************|
            elif prev in range(start_col, end_col) and current in range(start_col, end_col):
                spans.append(
                    TextSpan(
                        start=CharPos(row=current_row + n, col=0 + left_adjustment),
                        n_letters=line_len - 1,  # -1: dropped whitespace
                    )
                )

            elif prev > end_col:
                break

            prev = current

        return spans

    @staticmethod
    def _group_spans_by_row(blocks: Sequence[TextSpan]) -> Mapping[int, List[TextSpan]]:
        groups: Dict[int, List[TextSpan]] = {}
        for block in blocks:
            row = block.start.row
            if row in groups:
                groups[row].append(block)
            else:
                groups[row] = [block]
        return groups

    def __init__(self, sects={""}):
        HTMLParser.__init__(self)
        self.text = [""]
        self.ishead = False
        self.isinde = False
        self.isbull = False
        self.ispref = False
        self.ishidden = False
        self.idhead = set()
        self.idinde = set()
        self.idbull = set()
        self.idpref = set()
        self.idimgs = set()
        self.sects = sects
        self.sectsindex = {}
        self.italic_marks: List[TextMark] = []
        self.bold_marks: List[TextMark] = []
        self.imgs: Dict[int, str] = dict()

    def handle_starttag(self, tag, attrs):
        if re.match("h[1-6]", tag) is not None:
            self.ishead = True
        elif tag in self.inde:
            self.isinde = True
        elif tag in self.pref:
            self.ispref = True
        elif tag in self.bull:
            self.isbull = True
        elif tag in self.hide:
            self.ishidden = True
        elif tag == "sup":
            self.text[-1] += "^{"
        elif tag == "sub":
            self.text[-1] += "_{"
        # NOTE: "img" and "image"
        # In HTML, both are startendtag (no need endtag)
        # but in XHTML both need endtag
        elif tag in {"img", "image"}:
            for i in attrs:
                if (tag == "img" and i[0] == "src") or (tag == "image" and i[0].endswith("href")):
                    this_line = len(self.text)
                    self.idimgs.add(this_line)
                    self.imgs[this_line] = unquote(i[1])
                    self.text.append("[IMAGE]")
        # formatting
        elif tag in self.ital:
            if len(self.italic_marks) == 0 or self.italic_marks[-1].is_valid():
                char_pos = CharPos(row=len(self.text) - 1, col=len(self.text[-1]))
                self.italic_marks.append(TextMark(start=char_pos))
        elif tag in self.bold:
            if len(self.bold_marks) == 0 or self.bold_marks[-1].is_valid():
                char_pos = CharPos(row=len(self.text) - 1, col=len(self.text[-1]))
                self.bold_marks.append(TextMark(start=char_pos))
        if self.sects != {""}:
            for i in attrs:
                if i[0] == "id" and i[1] in self.sects:
                    # self.text[-1] += " (#" + i[1] + ") "
                    # self.sectsindex.append([len(self.text), i[1]])
                    self.sectsindex[len(self.text) - 1] = i[1]

    def handle_startendtag(self, tag, attrs):
        if tag == "br":
            self.text += [""]
        elif tag in {"img", "image"}:
            for i in attrs:
                #  if (tag == "img" and i[0] == "src")\
                #     or (tag == "image" and i[0] == "xlink:href"):
                if (tag == "img" and i[0] == "src") or (tag == "image" and i[0].endswith("href")):
                    this_line = len(self.text)
                    self.idimgs.add(this_line)
                    self.imgs[this_line] = unquote(i[1])
                    self.text.append("[IMAGE]")
                    self.text.append("")
        # sometimes attribute "id" is inside "startendtag"
        # especially html from mobi module (kindleunpack fork)
        if self.sects != {""}:
            for i in attrs:
                if i[0] == "id" and i[1] in self.sects:
                    # self.text[-1] += " (#" + i[1] + ") "
                    self.sectsindex[len(self.text) - 1] = i[1]

    def handle_endtag(self, tag):
        if re.match("h[1-6]", tag) is not None:
            self.text.append("")
            self.text.append("")
            self.ishead = False
        elif tag in self.para:
            self.text.append("")
        elif tag in self.hide:
            self.ishidden = False
        elif tag in self.inde:
            if self.text[-1] != "":
                self.text.append("")
            self.isinde = False
        elif tag in self.pref:
            if self.text[-1] != "":
                self.text.append("")
            self.ispref = False
        elif tag in self.bull:
            if self.text[-1] != "":
                self.text.append("")
            self.isbull = False
        elif tag in {"sub", "sup"}:
            self.text[-1] += "}"
        elif tag in {"img", "image"}:
            self.text.append("")
        # formatting
        elif tag in self.ital:
            char_pos = CharPos(row=len(self.text) - 1, col=len(self.text[-1]))
            last_mark = self.italic_marks[-1]
            self.italic_marks[-1] = dataclasses.replace(last_mark, end=char_pos)
        elif tag in self.bold:
            char_pos = CharPos(row=len(self.text) - 1, col=len(self.text[-1]))
            last_mark = self.bold_marks[-1]
            self.bold_marks[-1] = dataclasses.replace(last_mark, end=char_pos)

    def handle_data(self, raw):
        if raw and not self.ishidden:
            if self.text[-1] == "":
                tmp = raw.lstrip()
            else:
                tmp = raw
            if self.ispref:
                line = unescape(tmp)
            else:
                line = unescape(re.sub(r"\s+", " ", tmp))
            self.text[-1] += line
            if self.ishead:
                self.idhead.add(len(self.text) - 1)
            elif self.isbull:
                self.idbull.add(len(self.text) - 1)
            elif self.isinde:
                self.idinde.add(len(self.text) - 1)
            elif self.ispref:
                self.idpref.add(len(self.text) - 1)

    def get_structured_text(
        self, textwidth: Optional[int] = 0, starting_line: int = 0
    ) -> Union[Tuple[str, ...], TextStructure]:

        if not textwidth:
            return tuple(self.text)

        text: List[str] = []
        images: Dict[int, str] = dict()  # {line_num: path/in/zip}
        sect: Dict[str, int] = dict()  # {section_id: line_num}
        formatting: List[InlineStyle] = []

        italic_spans: List[TextSpan] = HTMLtoLines._mark_to_spans(self.text, self.italic_marks)
        bold_spans: List[TextSpan] = HTMLtoLines._mark_to_spans(self.text, self.bold_marks)
        italic_groups = HTMLtoLines._group_spans_by_row(italic_spans)
        bold_groups = HTMLtoLines._group_spans_by_row(bold_spans)

        for n, line in enumerate(self.text):

            startline = len(text)
            # findsect = re.search(r"(?<= \(#).*?(?=\) )", line)
            # if findsect is not None and findsect.group() in self.sects:
            # line = line.replace(" (#" + findsect.group() + ") ", "")
            # # line = line.replace(" (#" + findsect.group() + ") ", " "*(5+len(findsect.group())))
            # sect[findsect.group()] = len(text)
            if n in self.sectsindex.keys():
                sect[self.sectsindex[n]] = starting_line + len(text)
            if n in self.idhead:
                # text += [line.rjust(textwidth // 2 + len(line) // 2)] + [""]
                text += [line.center(textwidth)] + [""]
                formatting += [
                    InlineStyle(
                        row=starting_line + i, col=0, n_letters=len(text[i]), attr=self.attr_bold
                    )
                    for i in range(startline, len(text))
                ]
            elif n in self.idinde:
                text += ["   " + i for i in textwrap.wrap(line, textwidth - 3)] + [""]
            elif n in self.idbull:
                tmp = textwrap.wrap(line, textwidth - 3)
                text += [" - " + i if i == tmp[0] else "   " + i for i in tmp] + [""]
            elif n in self.idpref:
                tmp = line.splitlines()
                wraptmp = []
                for tmp_line in tmp:
                    wraptmp += [i for i in textwrap.wrap(tmp_line, textwidth - 6)]
                text += ["   " + i for i in wraptmp] + [""]
            elif n in self.idimgs:
                images[starting_line + len(text)] = self.imgs[n]
                text += [line.center(textwidth)]
                formatting += [
                    InlineStyle(
                        row=starting_line + len(text) - 1,
                        col=0,
                        n_letters=len(text[-1]),
                        attr=self.attr_bold,
                    )
                ]
                text += [""]
            else:
                text += textwrap.wrap(line, textwidth) + [""]

            endline = len(text)  # -1

            left_adjustment = 3 if n in self.idbull | self.idinde else 0

            for spans in italic_groups.get(n, []):
                italics = HTMLtoLines._adjust_wrapped_spans(
                    text[startline:endline],
                    spans,
                    line_adjustment=startline,
                    left_adjustment=left_adjustment,
                )
                for span in italics:
                    formatting.append(
                        InlineStyle(
                            row=starting_line + span.start.row,
                            col=span.start.col,
                            n_letters=span.n_letters,
                            attr=self.attr_italic,
                        )
                    )

            for spans in bold_groups.get(n, []):
                bolds = HTMLtoLines._adjust_wrapped_spans(
                    text[startline:endline],
                    spans,
                    line_adjustment=startline,
                    left_adjustment=left_adjustment,
                )
                for span in bolds:
                    formatting.append(
                        InlineStyle(
                            row=starting_line + span.start.row,
                            col=span.start.col,
                            n_letters=span.n_letters,
                            attr=self.attr_bold,
                        )
                    )

        # chapter suffix
        text += ["***".center(textwidth)]

        return TextStructure(
            text_lines=tuple(text),
            image_maps=images,
            section_rows=sect,
            formatting=tuple(formatting),
        )


def parse_html(
    html_src: str,
    *,
    textwidth: Optional[int] = None,
    section_ids: Optional[Set[str]] = None,
    starting_line: int = 0,
) -> Union[Tuple[str, ...], TextStructure]:
    """
    Parse html string into TextStructure

    :param html_src: html str to parse
    :param textwidth: textwidth to count max length of returned TextStructure
                      if None given, sequence of text as paragraph is returned
    :param section_ids: set of section ids to look for inside html tag attr
    :return: Tuple[str, ...] if textwidth not given else TextStructure
    """
    if not section_ids:
        section_ids = set()

    parser = HTMLtoLines(section_ids)
    # try:
    parser.feed(html_src)
    parser.close()
    # except:
    #     pass

    return parser.get_structured_text(textwidth, starting_line)