path: root/dlpcvp.py

#!/usr/bin/python3
import argparse
import configparser
import json
import logging
import os.path as osp
import sqlite3
import ssl
import sys
import urllib.request
import xml.etree.ElementTree as ET
from distutils.version import LooseVersion
from typing import Iterable, List, Optional, Tuple, Union
from urllib.error import URLError, HTTPError
from urllib.request import Request, urlopen

# PyPI API documentation https://warehouse.readthedocs.io/api-reference/
PyPI_base = "https://pypi.org/pypi/{}/json"
# https://github.com/openSUSE/open-build-service/blob/master/docs/api/api/api.txt
# https://build.opensuse.org/apidocs/index
OBS_base = "https://api.opensuse.org"
OBS_realm = "Use your novell account"
ConfigRCs = [osp.expanduser('~/.oscrc'),
             osp.expanduser('~/.config/osc/oscrc')]
CUTCHARS = len('python-')

DBConnType = sqlite3.Connection
OStr = Optional[str]

config = configparser.ConfigParser()
config.read(ConfigRCs)

logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s',
                    stream=sys.stdout, level=logging.INFO)
log = logging.getLogger()

# or HTTPPasswordMgrWithPriorAuth ?
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
user = config[OBS_base]['user']
passw = config[OBS_base]['pass']
password_mgr.add_password(OBS_realm, OBS_base, user, passw)

auth_handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
ssl_handler = urllib.request.HTTPSHandler(
    context=ssl._create_unverified_context())
opener = urllib.request.build_opener(auth_handler, ssl_handler)
urllib.request.install_opener(opener)

TB_EXISTS = """SELECT name FROM sqlite_master
        WHERE type='table' AND name='etags'
        """
# FIXME we need to distinguish between PyPI and OpenSUSE records
TB_CREATE = """CREATE TABLE etags
               (pkg text NOT NULL PRIMARY KEY,
               suse_name_etag text, suse_spec_etag text, pypi_etag text)
            """
TB_CREATE_IDX = "CREATE UNIQUE INDEX etags_names ON etags(pkg)"


def suse_packages(proj: str) -> Iterable[str]:
    """
    Iterator returning names of all packages in the given proj

    ETag management won't work here, because I don't know about any way
    how to return it in iterator.
    """
    req = Request(url=OBS_base + f'/source/{proj}')
    with open('exceptions.json') as exc_f:
        exc_dict = json.load(exc_f)

    exc_list = []  # type: List[str]
    if proj in exc_dict:
        exc_list = exc_dict[proj]

    try:
        with opener.open(req) as resp:
            raw_xml_data = ET.parse(resp)
            root = raw_xml_data.getroot()
            for elem in root.iter('entry'):
                pkg_name = elem.get('name')
                # Invalid packages are skipped
                if pkg_name is None:
                    log.warning(f"Package {elem} doesn’t have a name!")
                    continue
                # We don't want -doc subpackages
                if pkg_name.endswith('-doc'):
                    continue
                # Nor we want packages which we specifically excluded
                if pkg_name in exc_list:
                    continue
                yield pkg_name
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot find packages for {proj}!')
        else:
            raise


def get_etags(con: DBConnType, pkg: str) -> \
        Tuple[OStr, OStr, OStr, OStr]:
    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
    cur = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,))
    ret = cur.fetchone()
    log.debug(f'ret = {ret}')
    if ret is None:
        return None, None, None, None
    else:
        return ret


def update_etags(con: sqlite3.Connection, pkg: str,
                 e_suse_name: OStr, e_suse_spec: OStr, e_pypi: OStr):
    res = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone()
    if res:
        e_suse_name = res[1] if e_suse_name is None else e_suse_name
        e_suse_spec = res[2] if e_suse_spec is None else e_suse_spec
        e_pypi = res[3] if e_pypi is None else e_pypi
    con.execute('''REPLACE INTO etags
        (pkg, suse_name_etag, suse_spec_etag, pypi_etag)
        VALUES (?, ?, ?, ?)''', (pkg, e_suse_name, e_suse_spec, e_pypi))
    con.commit()


def is_develpackage(proj: str, pkg: str) -> bool:
    req = Request(url=OBS_base + f'/source/openSUSE:Factory/{pkg}/_meta')
    log.debug('Looking up URL: %s', req.full_url)
    try:
        with opener.open(req) as resp:
            xml_data = ET.parse(resp)
            for pel in xml_data.iter('package'):
                for delem in pel.iter('devel'):
                    if pel.attrib['name'] == pkg and \
                        delem.attrib['package'] == pkg and \
                            delem.attrib['project'] == proj:
                        return True
            return False
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot acquire _meta of {pkg}.')
            return None
        else:
            raise


def parse_spec(spec_file: Union[str, bytes]) -> LooseVersion:
    if isinstance(spec_file, bytes):
        spec_file = spec_file.decode()

    rest_of_line = ''
    for line in spec_file.splitlines():
        if line.startswith('Version:'):
            rest_of_line = line[len('Version:'):].strip()
            break

    return LooseVersion(rest_of_line)


def get_spec_name(req: Request, proj: str, pkg: str, etag: OStr = None) -> \
        OStr:
    """Acquire version from the listing of the project directory.
    """
    spec_files = []  # type: List[str]

    if etag is not None:
        req.add_header('ETag', etag)

    if not spec_files:
        IOError(f'Cannot find SPEC file for {pkg}.')

    try:
        with opener.open(req) as resp:
            etag = str(resp.info()['ETag'])
            raw_xml_data = ET.parse(resp)
            root = raw_xml_data.getroot()
            for elem in root.iter('entry'):
                name = elem.get('name')
                # Invalid package
                if name is None:
                    log.warning(f"Package {elem} doesn’t have a name!")
                    return None
                if name.endswith('.spec'):
                    spec_files.append(name)
                if name == '_aggregate':
                    log.warning(f'Package {pkg} is just aggregate, ignoring.')
                    return None

    except HTTPError as ex:
        if ex.getcode() in (400, 404):
            return None
        else:
            raise

    try:
        fname = sorted(spec_files, key=len)[0]
        return fname
    except IndexError:
        log.error(
                f'{pkg} is most likely not a branch, but a link: ' +
                f'use osc linktobranch {proj} {pkg} to convert it.')
        return None


def get_version_from_pypi(name: str, con: DBConnType = None) -> \
        Optional[LooseVersion]:
    """
    For the given name of module return the latest version available on PyPI.
    """
    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
    if con is None:
        etag = None
    else:
        _, _, _, etag = get_etags(con, name)
    req = Request(url=PyPI_base.format(name))

    if etag is not None:
        req.add_header('ETag', etag)

    try:
        with urlopen(req) as resp:
            data = json.load(resp)
            info_dict = data['info']
            curr_etag = str(resp.info()['ETag'])
            if (con is not None) and curr_etag:
                update_etags(con, name, None, None, curr_etag)

            # Cleanup version
            version = info_dict['version'].replace('-', '.')

            return LooseVersion(version)
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot find {name} on PyPI')
        else:
            raise
        return None


def package_version(proj: str, pkgn: str,
                    con: DBConnType = None) -> Optional[LooseVersion]:
    """
    Return the version of the given package in the given proj.

    Downloads SPEC file from OBS and parses it.
    """
    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
    if con:
        _, etag_fn, etag_spcf, _ = get_etags(con, pkgn)
    else:
        etag_fn, etag_spcf = None, None

    # Get listing of the package repository
    req_spc_name = Request(url=OBS_base + f'/source/{proj}/{pkgn}?expand=1')
    spc_fname = get_spec_name(req_spc_name, proj, pkgn, etag_fn)

    if spc_fname is None:
        return None

    req_spec = Request(url=OBS_base +
                       f'/source/{proj}/{pkgn}/{spc_fname}?expand=1')

    if etag_spcf is not None:
        req_spc_name.add_header('ETag', etag_spcf)

    try:
        with opener.open(req_spec) as resp:
            etag_spcf = str(resp.info()['ETag'])
            etag_spcf = None if not etag_spcf else etag_spcf
            spec_file_str = resp.read().decode()

            if (con is not None) and (etag_spcf or etag_fn):
                update_etags(con, pkgn, etag_fn, etag_spcf, None)
            return parse_spec(spec_file_str)
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot parse SPEC file {spc_fname} for {pkgn}')
        else:
            raise
        return None


def main(prj):
    db_name = osp.splitext(osp.basename(osp.realpath(__file__)))[0] + ".db"
    to_be_upgraded = []  # type: List[Tuple[str, LooseVersion, LooseVersion]]
    missing_on_PyPI = []  # type: List[str]
    with sqlite3.connect(db_name) as conn:
        if not conn.execute(TB_EXISTS).fetchone():
            conn.execute(TB_CREATE)
            conn.execute(TB_CREATE_IDX)

        for pkg in suse_packages(prj):
            log.debug(f'pkg = {pkg}')
            if pkg.startswith('python-'):
                pypi_name = pkg[CUTCHARS:]
                try:
                    suse_ver = package_version(prj, pkg, conn)
                    if suse_ver is None:
                        raise RuntimeError('not in OBS')
                    if not is_develpackage(prj, pkg):
                        continue
                    pypi_ver = get_version_from_pypi(pypi_name, conn)
                    if pypi_ver is None:
                        raise RuntimeError('not in PyPI')
                except RuntimeError as ex:
                    log.warning(f'Package {pkg} cannot be found: {ex}')
                    continue
                except URLError as ex:
                    log.warning(
                        'Investigation of package %s caused network error: %s',
                        pkg, ex)
                    continue

                try:
                    if pypi_ver > suse_ver:
                        to_be_upgraded.append((pkg, suse_ver, pypi_ver))
                except TypeError:
                    log.warning(f'{pkg} pypi_ver = {pypi_ver}')
                    log.warning(f'{pkg} suse_ver = {suse_ver}')
                    continue
            else:
                missing_on_PyPI.append(pkg)

    sys.stdout.flush()
    if missing_on_PyPI:
        print("\nThese packages don't seem to be available on PyPI:")
        print("{}\n".format('\n'.join(missing_on_PyPI)))

    if to_be_upgraded:
        print('These packages need to be upgraded:')
        for pkg_upgr in to_be_upgraded:
            print(f'{pkg_upgr[0]} ({pkg_upgr[1]} -> {pkg_upgr[2]})')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Check available versions '
                                     'of the upstream packages on PyPI')
    parser.add_argument('project', nargs='?',
                        default='devel:languages:python:numeric',
                        help='The OpenBuildService project. Defaults '
                        'to %(default)s')

    args = parser.parse_args()
    sys.exit(main(args.project))