path: root/dlpcvp.py

#!/usr/bin/python3
import argparse
import configparser
import json
import logging
import os.path as osp
import sqlite3
import ssl
import sys
import urllib.request
import xml.etree.ElementTree as ET
from distutils.version import LooseVersion
from enum import Enum, auto
from typing import Iterable, List, Optional, Tuple, Union
from urllib.error import URLError, HTTPError
from urllib.request import Request, urlopen

from thespian.actors import Actor, ActorSystem

# PyPI API documentation https://warehouse.readthedocs.io/api-reference/
PyPI_base = "https://pypi.org/pypi/{}/json"
# https://github.com/openSUSE/open-build-service/blob/master/docs/api/api/api.txt
# https://build.opensuse.org/apidocs/index
OBS_base = "https://api.opensuse.org"
OBS_realm = "Use your novell account"
ConfigRCs = [osp.expanduser('~/.oscrc'), osp.expanduser('~/.config/osc/oscrc')]
CUTCHARS = len('python-')

DBConnType = sqlite3.Connection
OStr = Optional[str]

config = configparser.ConfigParser()
config.read(ConfigRCs)

logging.basicConfig(
    format='%(levelname)s:%(funcName)s:%(message)s',
    stream=sys.stdout,
    level=logging.INFO,
)
log = logging.getLogger()

# or HTTPPasswordMgrWithPriorAuth ?
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
user = config[OBS_base]['user']
passw = config[OBS_base]['pass']
password_mgr.add_password(OBS_realm, OBS_base, user, passw)

auth_handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
ssl_handler = urllib.request.HTTPSHandler(
    context=ssl._create_unverified_context()
)
opener = urllib.request.build_opener(auth_handler, ssl_handler)
urllib.request.install_opener(opener)

TB_EXISTS = """SELECT name FROM sqlite_master
        WHERE type='table' AND name='etags'
        """
# FIXME we need to distinguish between PyPI and OpenSUSE records
TB_CREATE = """CREATE TABLE etags
               (pkg text NOT NULL PRIMARY KEY,
               suse_name_etag text, suse_spec_etag text, pypi_etag text)
            """
TB_CREATE_IDX = "CREATE UNIQUE INDEX etags_names ON etags(pkg)"


def suse_packages(proj: str) -> Iterable[str]:
    """
    Iterator returning names of all packages in the given proj

    ETag management won't work here, because I don't know about any way
    how to return it in iterator.
    """
    req = Request(url=OBS_base + f'/source/{proj}')
    with open('exceptions.json') as exc_f:
        exc_dict = json.load(exc_f)

    exc_list = []  # type: List[str]
    if proj in exc_dict:
        exc_list = exc_dict[proj]

    try:
        with opener.open(req) as resp:
            raw_xml_data = ET.parse(resp)
            root = raw_xml_data.getroot()
            for elem in root.iter('entry'):
                pkg_name = elem.get('name')
                # Invalid packages are skipped
                if pkg_name is None:
                    log.warning(f"Package {elem} doesn’t have a name!")
                    continue
                # We don't want -doc subpackages
                if pkg_name.endswith('-doc'):
                    continue
                # Nor we want packages which we specifically excluded
                if pkg_name in exc_list:
                    continue
                yield pkg_name
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot find packages for {proj}!')
        else:
            raise


def get_etags(con: DBConnType, pkg: str) -> Tuple[OStr, OStr, OStr, OStr]:
    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
    cur = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,))
    ret = cur.fetchone()
    log.debug(f'ret = {ret}')
    if ret is None:
        return None, None, None, None
    else:
        return ret


def update_etags(
    con: sqlite3.Connection,
    pkg: str,
    e_suse_name: OStr,
    e_suse_spec: OStr,
    e_pypi: OStr,
):
    res = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone()
    if res:
        e_suse_name = res[1] if e_suse_name is None else e_suse_name
        e_suse_spec = res[2] if e_suse_spec is None else e_suse_spec
        e_pypi = res[3] if e_pypi is None else e_pypi
    con.execute(
        '''REPLACE INTO etags
        (pkg, suse_name_etag, suse_spec_etag, pypi_etag)
        VALUES (?, ?, ?, ?)''',
        (pkg, e_suse_name, e_suse_spec, e_pypi),
    )
    con.commit()


def is_develpackage(proj: str, pkg: str) -> bool:
    req = Request(url=OBS_base + f'/source/openSUSE:Factory/{pkg}/_meta')
    log.debug('Looking up URL: %s', req.full_url)
    try:
        with opener.open(req) as resp:
            xml_data = ET.parse(resp)
            for pel in xml_data.iter('package'):
                for delem in pel.iter('devel'):
                    if (
                        pel.attrib['name'] == pkg
                        and delem.attrib['package'] == pkg
                        and delem.attrib['project'] == proj
                    ):
                        return True
            return False
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot acquire _meta of {pkg}.')
            return None
        else:
            raise


def parse_spec(spec_file: Union[str, bytes]) -> LooseVersion:
    if isinstance(spec_file, bytes):
        spec_file = spec_file.decode()

    rest_of_line = ''
    for line in spec_file.splitlines():
        if line.startswith('Version:'):
            rest_of_line = line[len('Version:') :].strip()
            break

    return LooseVersion(rest_of_line)


def get_spec_name(req: Request, proj: str, pkg: str, etag: OStr = None) -> OStr:
    """Acquire version from the listing of the project directory.
    """
    spec_files = []  # type: List[str]

    if etag is not None:
        req.add_header('ETag', etag)

    if not spec_files:
        IOError(f'Cannot find SPEC file for {pkg}.')

    try:
        with opener.open(req) as resp:
            etag = str(resp.info()['ETag'])
            raw_xml_data = ET.parse(resp)
            root = raw_xml_data.getroot()
            for elem in root.iter('entry'):
                name = elem.get('name')
                # Invalid package
                if name is None:
                    log.warning(f"Package {elem} doesn’t have a name!")
                    return None
                if name.endswith('.spec'):
                    spec_files.append(name)
                if name == '_aggregate':
                    log.warning(f'Package {pkg} is just aggregate, ignoring.')
                    return None

    except HTTPError as ex:
        if ex.getcode() in (400, 404):
            return None
        else:
            raise

    try:
        fname = sorted(spec_files, key=len)[0]
        return fname
    except IndexError:
        log.error(
            f'{pkg} is most likely not a branch, but a link: '
            + f'use osc linktobranch {proj} {pkg} to convert it.'
        )
        return None


def get_version_from_pypi(
    name: str, con: DBConnType = None
) -> Optional[LooseVersion]:
    """
    For the given name of module return the latest version available on PyPI.
    """
    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
    if con is None:
        etag = None
    else:
        _, _, _, etag = get_etags(con, name)
    req = Request(url=PyPI_base.format(name))

    if etag is not None:
        req.add_header('ETag', etag)

    try:
        with urlopen(req) as resp:
            data = json.load(resp)
            info_dict = data['info']
            curr_etag = str(resp.info()['ETag'])
            if (con is not None) and curr_etag:
                update_etags(con, name, None, None, curr_etag)

            # Cleanup version
            version = info_dict['version'].replace('-', '.')

            return LooseVersion(version)
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot find {name} on PyPI')
        else:
            raise
        return None


class SPkg(Enum):
    CURRENT = auto()
    NOOBS = auto()


class DispatchCmd(Enum):
    INIT = auto()
    PKG = auto()


class SPECParser(Actor):
    def package_version(
        self, proj: str, pkgn: str, con: DBConnType = None
    ) -> Optional[LooseVersion]:
        """
        Return the version of the given package in the given proj.

        Downloads SPEC file from OBS and parses it.
        """
        # pkg, suse_name_etag, suse_spec_etag, pypi_etag
        if con:
            _, etag_fn, etag_spcf, _ = get_etags(con, pkgn)
        else:
            etag_fn, etag_spcf = None, None

        # Get listing of the package repository
        req_spc_name = Request(url=OBS_base +
                               f'/source/{proj}/{pkgn}?expand=1')
        spc_fname = get_spec_name(req_spc_name, proj, pkgn, etag_fn)

        if spc_fname is None:
            return None

        req_spec = Request(
            url=OBS_base + f'/source/{proj}/{pkgn}/{spc_fname}?expand=1'
        )

        if etag_spcf is not None:
            req_spc_name.add_header('ETag', etag_spcf)

        try:
            with opener.open(req_spec) as resp:
                etag_spcf = str(resp.info()['ETag'])
                etag_spcf = None if not etag_spcf else etag_spcf
                spec_file_str = resp.read().decode()

                if (con is not None) and (etag_spcf or etag_fn):
                    update_etags(con, pkgn, etag_fn, etag_spcf, None)
                return parse_spec(spec_file_str)
        except HTTPError as ex:
            if ex.getcode() == 404:
                log.warning(f'Cannot parse SPEC file {spc_fname} for {pkgn}')
            else:
                raise
            return SPkg.NOOBS

    def receiveMessage(self, msg, sender):
        pass


class PkgDispatcher(Actor):
    def __init__(self, conn, project):
        self.conn = None  # database connection for ETags management
        self.prj = None   # currently processed project on OBS

    def receiveMessage(self, msg, sender):
        cmd, val = msg

        if cmd is DispatchCmd.INIT:
            self.conn = val.conn
            self.prj = val.prj
        elif cmd is DispatchCmd.PKG:
            pkg = msg
            log.debug('pkg = %s', pkg)
            print(pkg[0], file=sys.stderr, end='', flush=True)
            if pkg.startswith('python-'):
                pypi_name = pkg[CUTCHARS:]
                try:
                    suse_ver = self.package_version(self.prj, pkg, self.conn)
                    if suse_ver is SPkg.NOOBS:
                        return suse_ver
                    if not is_develpackage(prj, pkg):
                        continue
                    pypi_ver = get_version_from_pypi(pypi_name, conn)
                    if pypi_ver is None:
                        raise RuntimeError('not in PyPI')
                except RuntimeError as ex:
                    log.warning('Package %s cannot be found: %s', pkg, ex)
                    continue
                except URLError as ex:
                    log.warning(
                        'Investigation of package %s caused network error: %s',
                        pkg, ex)
                    continue

                try:
                    if pypi_ver > suse_ver:
                        to_be_upgraded.append((pkg, suse_ver, pypi_ver))
                except TypeError:
                    log.warning('%s pypi_ver = %s', pkg, pypi_ver)
                    log.warning('%s suse_ver = %s', pkg, suse_ver)
                    continue
            else:
                missing_on_PyPI.append(pkg)


def main(prj):
    db_name = osp.splitext(osp.basename(osp.realpath(__file__)))[0] + ".db"
    to_be_upgraded = []  # type: List[Tuple[str, LooseVersion, LooseVersion]]
    missing_on_PyPI = []  # type: List[str]
    actorsys = ActorSystem()

    with sqlite3.connect(db_name) as conn:
        if not conn.execute(TB_EXISTS).fetchone():
            conn.execute(TB_CREATE)
            conn.execute(TB_CREATE_IDX)

        act_dispatcher = actorsys.createActor(PkgDispatcher)  # prj conn
        actorsys.tell(act_dispatcher, (DispatchCmd.INIT, (conn, prj)))

        # https://thespianpy.com/doc/in_depth#outline-container-orgbc15e0e
        # There are no futures in the Actors model, and there shouldn't
        # be ones:
        # Actors thus operate on individual messages, running when
        # a message is received and finishing when they exit from the
        # receiveMessage(), during which they may have created other
        # Actors or sent other messages. This message passing paradigm
        # is different from the conventional blocking paradigm used by
        # most code. In Thespian, there are no need for special objects
        # like futures that hide asynchronous functionality behind what
        # looks like synchronous code. If an Actor does block during the
        # receiveMessage() execution, it will not receive other messages
        # while it is blocked; the typical method of addressing this in
        # an Actor model is to start up more Actors or delegate the
        # blocking functionality to a sub-Actor that was dynamically
        # created as needed.
        # TL;DR: Individual actors just have to send message to
        # appropriate actors processing results.
        for pkg in suse_packages(prj):
            ret = actorsys.ask(act_dispatcher, (DispatchCmd.PKG, pkg))

    actorsys.shutdown()  # end of the processing

    sys.stdout.flush()
    if missing_on_PyPI:
        print("\nThese packages don't seem to be available on PyPI:")
        print('{}\n'.format('\n'.join(missing_on_PyPI)))

    if to_be_upgraded:
        print('These packages need to be upgraded:')
        for pkg_upgr in to_be_upgraded:
            print(f'{pkg_upgr[0]} ({pkg_upgr[1]} -> {pkg_upgr[2]})')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Check available versions '
        'of the upstream packages on PyPI'
    )
    parser.add_argument(
        'project',
        nargs='?',
        default='devel:languages:python:numeric',
        help='The OpenBuildService project. Defaults ' 'to %(default)s',
    )

    args = parser.parse_args()
    sys.exit(main(args.project))