path: root/modules/conf/confmaker.py

#!/usr/bin/env python3

# -*- coding: utf-8 -*-
# confmaker.py - Provides a initial conf file for a new module by analyzing
#                the related OSIS xml file.

## The programme searches for relevant tags and creates the GlobalOptionFilter
#  entries and other relevant conf entries. This a port to Python from the
#  previous confmaker.pl Perl script we were using. It fixes detection of
#  diacritics and OSISMorphSegmentation (GlobalOpionFilters) and adds support
#  for genbook and modules with large entries > 64Kb.

# Copyright (C) 2020 CrossWire Bible Society


# Author: kris <kristof.szabo@lutheran.hu> & domcox <domcox@crosswire.org>

# This file is part of Sword Modules

# Sword Modules is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# Sword Modules is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with Sword Modules.  If not, see <https://www.gnu.org/licenses/>.

# Created:  2021-01-08
#
# Revision:
# 2021-01-16 domcox <domcox@crosswire.org>
#            Changed language library from iso-639 to langtags
# 2023-07-30 domcox <domcox@crosswire.org>
#            Full rewrite using ElementTree XML parsing module


# TODO:
# - EntrySize for verses that do not use milestone elements
# - EntrySize for book titles & introduction
# - Implement GlobalOptionsFilter=OSISReferenceLinks

import argparse
import sys
import time
import xml.etree.ElementTree as ET
from datetime import date
from pathlib import Path
try:
     import langtags
except:
     sys.stderr.write("You do not have the Python langtags library installed. Please install it (pip install langtags).\n")
     sys.exit(1)
try:
     import Sword
except:
     sys.stderr.write("You do not have the SWORD library installed. Please install it.\n")
     sys.exit(1)


# Variables

Version = '2.0'

# List of V11n and relative SWORD Minimum version
versification = {
    'KJV': '1.5.9',
    'KJVA': '1.6.0',
    'NRSV': '1.6.0',
    'NRSVA': '1.6.0',
    'MT': '1.6.0',
    'Leningrad': '1.6.0',
    'Synodal': '1.6.1',
    'Vulg': '1.6.1',
    'Luther': '1.6.1',
    'German': '1.6.1',
    'Catholic': '1.6.2',
    'Catholic2': '1.6.2',
    'LXX': '1.7.2',
    'Orthodox': '1.7.2',
    'SynodalProt': '1.7.2',
    'DarbyFr': '1.8.0',
    'Segond': '1.8.0',
    'Calvin': '1.8.0'
}

# Functions

def die(msg):
     """
     Show an error message then exit on error
     """
     print('ERROR! ' + msg, file=sys.stderr)
     sys.exit(1)


def get_parameters():
    """
    Get Parse command-line options.
    Returns dict containing parameters values
    """

    # Creating parser
    description = '''
    provides a conf file for a module by analysing the given OSIS XML file.
    Optionally include extra elements from a conf.in file. This option will
    be removed in a future version.
    '''
    parser = argparse.ArgumentParser(description=description)

    # Adding arguments
    parser.add_argument('osis',
               help='name of the OSIS XML file')
    parser.add_argument("-o", "--outfile",
               help="name of generated conf file, (default to screen)")
    parser.add_argument("-v", "--v11n", default='KJV',
               help="versification schema, (default: KJV)")
    parser.add_argument("-s", "--size", default='2',
               help="set -s 4 for modules with large entries > 64Kb, (default -s 2)")
    parser.add_argument("-i", "--infile",
               help="conf.in file containing extra elements to include, (default none)")

    # Parsing arguments
    args = parser.parse_args()

    return (vars(args))


def check_parameters(params):
     """
     Check command arguments
     """

     # Check OSIS file value
     osisfile = params['osis']
     fileObj = Path(osisfile)
     if not fileObj.is_file():
          die(f"File '{osisfile}' does not exist.")

     # Check conf.in file value
     if params['infile']:
          infile = params['infile']
          fileObj = Path(infile)
          if not fileObj.is_file():
               die(f"File '{infile}' does not exist.")

     # Check versification schema
     v11n = params['v11n']
     av11n = versification.keys()
     if v11n not in av11n:
          die(f"'{v11n}': Unknown versification schema.")
     return (True)


def get_language(lang):
    """
    Search BCP-47 Languages Database for the given lang
    """

    found = False

    try:
        tag = langtags.Tag(lang)
        found = True
    except:
         die(f"Language '{lang}' not found in BCP 47 Languages Database")

    # Sometimes language description is multiline -> remove '\n'
    return (tag.language.description.replace('\n', ' '))


def is_diacritic(xml_file, lang, diacritic):
     '''
     Search for 'diacritic' in OSIS File
     Returns True or False
     '''
     # Don't search OSIS targetting other languages than Hebrew, Greek, Arabic
     if not lang in ('ar','grc','he','hbo'):
          return False
     elif not lang in 'ar' and diacritic == 'Arabic Vowel Points':
          return False
     elif not lang in 'grc' and diacritic == 'Greek Accents':
          return False
     elif not lang in ('he','hbo') and diacritic == 'Hebrew Cantillation':
          return False
     elif not lang in ('he','hbo') and diacritic == 'Hebrew Vowel Points':
          return False
     else:
          # Grab the base SWORD manager
          mgr = Sword.SWMgr()
          mgr.setGlobalOption("Arabic Vowel Points", "Off");
          mgr.setGlobalOption("Greek Accents", "Off");
          mgr.setGlobalOption("Hebrew Cantillation", "Off");
          mgr.setGlobalOption("Hebrew Vowel Points", "Off");

          # Parse XML
          xml_text = ET.parse(xml_file)
          xml_root = xml_text.getroot()

          # Remove all tags and keep bare text only, make 2 sets
          strip_text = ET.tostring(xml_root, encoding='unicode', method='text')
          ref_text = Sword.SWBuf(strip_text)
          mod_text = Sword.SWBuf(strip_text)

          # Apply filter on 1 text
          mgr.filterText(diacritic, mod_text)

          # Compare original bare text and filtered one
          # return True is the filter has made changes to the text, False otherwise
          return(ref_text.c_str() != mod_text.c_str())


def osis2conf_parser(args):
     '''
     This function Parses the OSIS file. searches for specific tags
     and creates the relevant conf elements that will be used to build the conf file.
     '''
     # Variables:

     # 1. List of Key elements of the resulting SWORD conf file
     Elements = []

     # 2. OSIS sections
     Header = False
     Chapter = False
     Verse = False

     # 3. Big entry
     if int(args['size']) > 2:
          EntrySize = 655536
     else:
          EntrySize=0

     # 4. Key elements of a SWORD module.conf
     Name = ''
     Type = ''
     Lang = ''
     Description = ''
     About = ''
     TextSource = ''
     DistributionLicense = ''
     OSISFootnotes = False
     OSISHeadings = False
     OSISScripref = False
     OSISRedLetterWords = False
     OSISVariants = False
     OSISMorphSegmentation = False
     OSISLemma = False
     OSISStrongs = False
     OSISGlosses = False
     OSISMorph = False
     OSISEnum = False
     OSISXlit = False
     Images = False
     NoParagraphs = True
     Copyright = ''
     CopyrightHolder = ''
     CopyrightDate = ''
     CopyrightNotes = ''
     CopyrightContactName = ''
     CopyrightContactNotes = ''
     CopyrightContactAddress = ''
     Abbreviation = ''
     KeyType = ''
     DisplayLevel = ''
     CaseSensitiveKeys=''
     PreferredCSSXHTML = ''
     Obsoletes = ''
     Companion = ''

     # Let's parse
     for event, node in ET.iterparse(args['osis'], events=("start", "end")):
          # OsisText content
          if not Name:
               if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}osisText':
                    # Get osisIDWork
                    Name = node.get('osisIDWork')
                    if not Name:
                         die('FATAL: osisIDWork is empty.')
                    # Get osisRefWork
                    Type = node.get('osisRefWork').lower()
                    if Type not in ['bible', 'commentary', 'genbook']:
                         die(f"FATAL: Invalid attribute osisRefWork: {osiswork}")
                    # Get Language
                    Lang = node.get('{http://www.w3.org/XML/1998/namespace}lang')
                    if not Lang:
                         die(f'FATAL: Missing lang element')

          # Select header
          if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}header' and event == "start":
               Header = True
          if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}header' and event == "end":
               Header = False
          # Select Chapter
          if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}chapter' and event == "start":
               Chapter = True
          if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}chapter' and event == "end":
               Chapter = False

          # GlobalOptionFilters
          if not Header:
               # Footnotes
               if not OSISFootnotes:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}note':
                         OSISFootnotes = True
               # Headings
               if not OSISHeadings:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}title':
                         OSISHeadings = True
               # Scripref
               if not OSISScripref:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}reference':
                         OSISScripref = True
               # RedLetterWords
               if not OSISRedLetterWords:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}q':
                         OSISRedLetterWords = True
               # Variants
               if not OSISVariants:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}seg':
                         if 'type' in node.keys():
                              if 'x-variant' in node.get('type'):
                                   OSISVariants = True
               # MorphSegmentation
               if not OSISMorphSegmentation:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}seg':
                         if 'type' in node.keys():
                              if 'morph:' in node.get('type'):
                                   OSISMorphSegmentation = True
               # Lemma
               if not OSISLemma:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}w':
                         if node.get('lemma') != None:
                              OSISLemma = True
               # Strongs
               if not OSISStrongs:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}w':
                         if 'lemma' in node.keys():
                              if 'strong' in node.get('lemma'):
                                   OSISStrongs = True
               # Glosses
               if not OSISGlosses:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}w':
                         if 'gloss' in node.keys():
                              OSISGlosses = True
               # Morph
               if not OSISMorph:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}w':
                         if 'morph' in node.keys():
                              OSISMorph = True
               # Enum
               if not OSISEnum:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}w':
                         if 'n' in node.keys():
                              OSISEnum = True
               # Xlit
               if not OSISXlit:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}w':
                         if 'xlit' in node.keys():
                              OSISXlit = True

               # Images
               if not Images:
                    if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}figure':
                         Images = True
               # Search only inside Chapters
               if Chapter:
                    # NoParagraphs
                    if NoParagraphs:
                         if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}p':
                              NoParagraphs = False
               # Entries length - Get verse max size
               if node.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}verse':
                    if 'sID' in node.keys():
                         Verse = True
                         rawtext = ''
                    if 'eID' in node.keys():
                         Verse = False
                         if len(rawtext) > EntrySize:
                              EntrySize =len(rawtext)
               # Entries length - Get verse text
               if Verse:
                    if node.text != None:
                         rawtext = rawtext + node.text
                    if node.tail != None:
                         rawtext = rawtext + node.tail

     # Define Elements
     # Set Name
     Elements.append("[" + Name + "]")
     # Derive module name
     module = Name.lower()
     # Set big entry option for entries greater than 64K bytes
     big = ''
     if EntrySize > 64000:
          big='4'
     # Set ModDrv
     if Type in 'bible':
          Elements.append("ModDrv=zText" + big)
     if Type in 'commentary':
          Elements.append("ModDrv=zCom" + big)
     if Type in 'genbook':
          Elements.append("ModDrv=RawGenBook" + big)
     # Set Datapath
     if Type in 'bible':
          Elements.append("DataPath=./modules/texts/ztext" + big + "/" + module + "/")
     if Type in 'commentary':
          Elements.append("DataPath=./modules/comments/zcom" + big + "/" + module + "/")
     if Type in 'genbook':
          Elements.append("DataPath=./modules/genbook/rawgenbook/" + big + "/" + module + "/" + module)
     # Set Compression
     if Type in ['bible', 'commentary']:
          Elements.append('CompressType=ZIP')
     # Set misc. elements
     Elements.append('BlockType=BOOK')
     Elements.append('Encoding=UTF-8')
     Elements.append('SourceType=OSIS')
     Elements.append('OSISVersion=2.1.1')
     Elements.append('SwordVersionDate=' + str(date.today()))

     # Set Lang
     Elements.append('Lang=' + Lang)
     # Set GlobalOptionFilters
     if OSISFootnotes:
          Elements.append('GlobalOptionFilter=OSISFootnotes')
     if OSISHeadings:
          Elements.append('GlobalOptionFilter=OSISHeadings')
     if OSISScripref:
          Elements.append('GlobalOptionFilter=OSISScripref')
     if OSISRedLetterWords:
          Elements.append('GlobalOptionFilter=OSISRedLetterWords')
     if OSISVariants:
          Elements.append('GlobalOptionFilter=OSISVariants')
     if OSISMorphSegmentation:
          Elements.append('GlobalOptionFilter=OSISMorphSegmentation')
     if OSISLemma:
          Elements.append('GlobalOptionFilter=OSISLemma')
     if OSISStrongs:
          Elements.append('GlobalOptionFilter=OSISStrongs')
     if OSISGlosses:
          Elements.append('GlobalOptionFilter=OSISGlosses')
     if OSISMorph:
          Elements.append('GlobalOptionFilter=OSISMorph')
     if OSISEnum:
          Elements.append('GlobalOptionFilter=OSISEnum')
     if OSISXlit:
          Elements.append('GlobalOptionFilter=OSISXlit')
     # Set Diacritics
     # Hebrew Vowel Points
     if is_diacritic(args['osis'], Lang, 'Hebrew Vowel Points'):
          Elements.append('GlobalOptionFilter=UTF8HebrewPoints')
     # Arabic Vowel Points
     if is_diacritic(args['osis'], Lang, 'Arabic Vowel Points'):
          Elements.append('GlobalOptionFilter=UTF8ArabicPoints')
     # Hebrew Cantillation
     if is_diacritic(args['osis'], Lang, 'Hebrew Cantillation'):
          Elements.append('GlobalOptionFilter=UTF8Cantillation')
     # Greek Accents
     if is_diacritic(args['osis'], Lang, 'Greek Accents'):
          Elements.append('GlobalOptionFilter=UTF8GreekAccents ')
     # Set Features
     if OSISStrongs:
          Elements.append('Feature=StrongsNumbers')
     # Images
     if Images:
          Elements.append('Feature=Images')
     # NoParagraphs
     if NoParagraphs:
          Elements.append('Feature=NoParagraphs')
     # Set LCSH
     language = get_language(Lang)
     if Type not in 'genbook':
          Elements.append('LCSH=' + Type.capitalize() + '.' + language)

     # Set Sword Minimum Version
     Elements.append('MinimumVersion=' + versification[args['v11n']])
     # Set Versification
     if Type not in 'genbook':
          Elements.append('Versification=' + args['v11n'])

     # End
     print('EntrySize=',EntrySize)
     return Elements


def main():
     '''
     Main function
     '''
      # Start benchmark
     start_time = time.perf_counter()

     # Read CLI params
     params = get_parameters()
     check_parameters(params)

     # Parse OSIS
     conf = osis2conf_parser(params)
     # print('conf=', conf)

     # Generate conf file
     outfile = params['outfile']
     if not outfile:
          # Default to screen
          for key in conf:
               print(key)
     else:
          # Write config to file
          with open(outfile, 'w') as f:
               for key in conf:
                    print(key, file=f)

     # Include conf.in file if it exists
     infile = params['infile']
     if infile:
          # Read and include conf.in contents
          with open(infile, 'r', encoding='utf-8', newline='\n') as f:
               for line in f:
                    if not outfile:
                         # Default to screen
                         print(line.rstrip())
                    else:
                         # Write config to file
                         with open(outfile, 'a') as f:
                              print(line.rstrip(), file=f)
     else:
          # No conf.in file -> generate default values
          if not outfile:
               # Default to screen
               print('DistributionLicense=Copyrighted')
               print('Description=This is a new module')
               print('Version=1.0')
               print('History_1.0=First release')
          else:
               # Write config to file
               with open(outfile, 'a') as f:
                    print('DistributionLicense=Copyrighted', file=f)
                    print('Description=This is a new module', file=f)
                    print('Version=1.0', file=f)
                    print('History_1.0=First release', file=f)

     # Benchmark results
     end_time = time.perf_counter()
     total_time = round(end_time - start_time, 1)
     print(f'-- Module Config generated in {total_time} s')

main()