#!/usr/bin/python3
import argparse
import configparser
import json
import logging
import os.path as osp
import sqlite3
import ssl
import sys
import urllib.request
import xml.etree.ElementTree as ET
from distutils.version import LooseVersion
from enum import Enum, auto
from typing import Iterable, List, Optional, Tuple, Union
from urllib.error import URLError, HTTPError
from urllib.request import Request, urlopen
from thespian.actors import Actor, ActorSystem
# PyPI API documentation https://warehouse.readthedocs.io/api-reference/
PyPI_base = "https://pypi.org/pypi/{}/json"
# https://github.com/openSUSE/open-build-service/blob/master/docs/api/api/api.txt
# https://build.opensuse.org/apidocs/index
OBS_base = "https://api.opensuse.org"
OBS_realm = "Use your novell account"
ConfigRCs = [osp.expanduser('~/.oscrc'), osp.expanduser('~/.config/osc/oscrc')]
CUTCHARS = len('python-')
DBConnType = sqlite3.Connection
OStr = Optional[str]
config = configparser.ConfigParser()
config.read(ConfigRCs)
logging.basicConfig(
format='%(levelname)s:%(funcName)s:%(message)s',
stream=sys.stdout,
level=logging.INFO,
)
log = logging.getLogger()
# or HTTPPasswordMgrWithPriorAuth ?
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
user = config[OBS_base]['user']
passw = config[OBS_base]['pass']
password_mgr.add_password(OBS_realm, OBS_base, user, passw)
auth_handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
ssl_handler = urllib.request.HTTPSHandler(
context=ssl._create_unverified_context()
)
opener = urllib.request.build_opener(auth_handler, ssl_handler)
urllib.request.install_opener(opener)
TB_EXISTS = """SELECT name FROM sqlite_master
WHERE type='table' AND name='etags'
"""
# FIXME we need to distinguish between PyPI and OpenSUSE records
TB_CREATE = """CREATE TABLE etags
(pkg text NOT NULL PRIMARY KEY,
suse_name_etag text, suse_spec_etag text, pypi_etag text)
"""
TB_CREATE_IDX = "CREATE UNIQUE INDEX etags_names ON etags(pkg)"
def suse_packages(proj: str) -> Iterable[str]:
"""
Iterator returning names of all packages in the given proj
ETag management won't work here, because I don't know about any way
how to return it in iterator.
"""
req = Request(url=OBS_base + f'/source/{proj}')
with open('exceptions.json') as exc_f:
exc_dict = json.load(exc_f)
exc_list = [] # type: List[str]
if proj in exc_dict:
exc_list = exc_dict[proj]
try:
with opener.open(req) as resp:
raw_xml_data = ET.parse(resp)
root = raw_xml_data.getroot()
for elem in root.iter('entry'):
pkg_name = elem.get('name')
# Invalid packages are skipped
if pkg_name is None:
log.warning(f"Package {elem} doesn’t have a name!")
continue
# We don't want -doc subpackages
if pkg_name.endswith('-doc'):
continue
# Nor we want packages which we specifically excluded
if pkg_name in exc_list:
continue
yield pkg_name
except HTTPError as ex:
if ex.getcode() == 404:
log.warning(f'Cannot find packages for {proj}!')
else:
raise
def get_etags(con: DBConnType, pkg: str) -> Tuple[OStr, OStr, OStr, OStr]:
# pkg, suse_name_etag, suse_spec_etag, pypi_etag
cur = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,))
ret = cur.fetchone()
log.debug(f'ret = {ret}')
if ret is None:
return None, None, None, None
else:
return ret
def update_etags(
con: sqlite3.Connection,
pkg: str,
e_suse_name: OStr,
e_suse_spec: OStr,
e_pypi: OStr,
):
res = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone()
if res:
e_suse_name = res[1] if e_suse_name is None else e_suse_name
e_suse_spec = res[2] if e_suse_spec is None else e_suse_spec
e_pypi = res[3] if e_pypi is None else e_pypi
con.execute(
'''REPLACE INTO etags
(pkg, suse_name_etag, suse_spec_etag, pypi_etag)
VALUES (?, ?, ?, ?)''',
(pkg, e_suse_name, e_suse_spec, e_pypi),
)
con.commit()
def is_develpackage(proj: str, pkg: str) -> bool:
req = Request(url=OBS_base + f'/source/openSUSE:Factory/{pkg}/_meta')
log.debug('Looking up URL: %s', req.full_url)
try:
with opener.open(req) as resp:
xml_data = ET.parse(resp)
for pel in xml_data.iter('package'):
for delem in pel.iter('devel'):
if (
pel.attrib['name'] == pkg
and delem.attrib['package'] == pkg
and delem.attrib['project'] == proj
):
return True
return False
except HTTPError as ex:
if ex.getcode() == 404:
log.warning(f'Cannot acquire _meta of {pkg}.')
return None
else:
raise
def parse_spec(spec_file: Union[str, bytes]) -> LooseVersion:
if isinstance(spec_file, bytes):
spec_file = spec_file.decode()
rest_of_line = ''
for line in spec_file.splitlines():
if line.startswith('Version:'):
rest_of_line = line[len('Version:') :].strip()
break
return LooseVersion(rest_of_line)
def get_spec_name(req: Request, proj: str, pkg: str, etag: OStr = None) -> OStr:
"""Acquire version from the listing of the project directory.
"""
spec_files = [] # type: List[str]
if etag is not None:
req.add_header('ETag', etag)
if not spec_files:
IOError(f'Cannot find SPEC file for {pkg}.')
try:
with opener.open(req) as resp:
etag = str(resp.info()['ETag'])
raw_xml_data = ET.parse(resp)
root = raw_xml_data.getroot()
for elem in root.iter('entry'):
name = elem.get('name')
# Invalid package
if name is None:
log.warning(f"Package {elem} doesn’t have a name!")
return None
if name.endswith('.spec'):
spec_files.append(name)
if name == '_aggregate':
log.warning(f'Package {pkg} is just aggregate, ignoring.')
return None
except HTTPError as ex:
if ex.getcode() in (400, 404):
return None
else:
raise
try:
fname = sorted(spec_files, key=len)[0]
return fname
except IndexError:
log.error(
f'{pkg} is most likely not a branch, but a link: '
+ f'use osc linktobranch {proj} {pkg} to convert it.'
)
return None
def get_version_from_pypi(
name: str, con: DBConnType = None
) -> Optional[LooseVersion]:
"""
For the given name of module return the latest version available on PyPI.
"""
# pkg, suse_name_etag, suse_spec_etag, pypi_etag
if con is None:
etag = None
else:
_, _, _, etag = get_etags(con, name)
req = Request(url=PyPI_base.format(name))
if etag is not None:
req.add_header('ETag', etag)
try:
with urlopen(req) as resp:
data = json.load(resp)
info_dict = data['info']
curr_etag = str(resp.info()['ETag'])
if (con is not None) and curr_etag:
update_etags(con, name, None, None, curr_etag)
# Cleanup version
version = info_dict['version'].replace('-', '.')
return LooseVersion(version)
except HTTPError as ex:
if ex.getcode() == 404:
log.warning(f'Cannot find {name} on PyPI')
else:
raise
return None
class SPkg(Enum):
CURRENT = auto()
NOOBS = auto()
class DispatchCmd(Enum):
INIT = auto()
PKG = auto()
class SPECParser(Actor):
def package_version(
self, proj: str, pkgn: str, con: DBConnType = None
) -> Optional[LooseVersion]:
"""
Return the version of the given package in the given proj.
Downloads SPEC file from OBS and parses it.
"""
# pkg, suse_name_etag, suse_spec_etag, pypi_etag
if con:
_, etag_fn, etag_spcf, _ = get_etags(con, pkgn)
else:
etag_fn, etag_spcf = None, None
# Get listing of the package repository
req_spc_name = Request(url=OBS_base +
f'/source/{proj}/{pkgn}?expand=1')
spc_fname = get_spec_name(req_spc_name, proj, pkgn, etag_fn)
if spc_fname is None:
return None
req_spec = Request(
url=OBS_base + f'/source/{proj}/{pkgn}/{spc_fname}?expand=1'
)
if etag_spcf is not None:
req_spc_name.add_header('ETag', etag_spcf)
try:
with opener.open(req_spec) as resp:
etag_spcf = str(resp.info()['ETag'])
etag_spcf = None if not etag_spcf else etag_spcf
spec_file_str = resp.read().decode()
if (con is not None) and (etag_spcf or etag_fn):
update_etags(con, pkgn, etag_fn, etag_spcf, None)
return parse_spec(spec_file_str)
except HTTPError as ex:
if ex.getcode() == 404:
log.warning(f'Cannot parse SPEC file {spc_fname} for {pkgn}')
else:
raise
return SPkg.NOOBS
def receiveMessage(self, msg, sender):
pass
class PkgDispatcher(Actor):
def __init__(self, conn, project):
self.conn = None # database connection for ETags management
self.prj = None # currently processed project on OBS
def receiveMessage(self, msg, sender):
cmd, val = msg
if cmd is DispatchCmd.INIT:
self.conn = val.conn
self.prj = val.prj
elif cmd is DispatchCmd.PKG:
pkg = msg
log.debug('pkg = %s', pkg)
print(pkg[0], file=sys.stderr, end='', flush=True)
if pkg.startswith('python-'):
pypi_name = pkg[CUTCHARS:]
try:
suse_ver = self.package_version(self.prj, pkg, self.conn)
if suse_ver is SPkg.NOOBS:
return suse_ver
if not is_develpackage(prj, pkg):
continue
pypi_ver = get_version_from_pypi(pypi_name, conn)
if pypi_ver is None:
raise RuntimeError('not in PyPI')
except RuntimeError as ex:
log.warning('Package %s cannot be found: %s', pkg, ex)
continue
except URLError as ex:
log.warning(
'Investigation of package %s caused network error: %s',
pkg, ex)
continue
try:
if pypi_ver > suse_ver:
to_be_upgraded.append((pkg, suse_ver, pypi_ver))
except TypeError:
log.warning('%s pypi_ver = %s', pkg, pypi_ver)
log.warning('%s suse_ver = %s', pkg, suse_ver)
continue
else:
missing_on_PyPI.append(pkg)
def main(prj):
db_name = osp.splitext(osp.basename(osp.realpath(__file__)))[0] + ".db"
to_be_upgraded = [] # type: List[Tuple[str, LooseVersion, LooseVersion]]
missing_on_PyPI = [] # type: List[str]
actorsys = ActorSystem()
with sqlite3.connect(db_name) as conn:
if not conn.execute(TB_EXISTS).fetchone():
conn.execute(TB_CREATE)
conn.execute(TB_CREATE_IDX)
act_dispatcher = actorsys.createActor(PkgDispatcher) # prj conn
actorsys.tell(act_dispatcher, (DispatchCmd.INIT, (conn, prj)))
# https://thespianpy.com/doc/in_depth#outline-container-orgbc15e0e
# There are no futures in the Actors model, and there shouldn't
# be ones:
# Actors thus operate on individual messages, running when
# a message is received and finishing when they exit from the
# receiveMessage(), during which they may have created other
# Actors or sent other messages. This message passing paradigm
# is different from the conventional blocking paradigm used by
# most code. In Thespian, there are no need for special objects
# like futures that hide asynchronous functionality behind what
# looks like synchronous code. If an Actor does block during the
# receiveMessage() execution, it will not receive other messages
# while it is blocked; the typical method of addressing this in
# an Actor model is to start up more Actors or delegate the
# blocking functionality to a sub-Actor that was dynamically
# created as needed.
# TL;DR: Individual actors just have to send message to
# appropriate actors processing results.
for pkg in suse_packages(prj):
ret = actorsys.ask(act_dispatcher, (DispatchCmd.PKG, pkg))
actorsys.shutdown() # end of the processing
sys.stdout.flush()
if missing_on_PyPI:
print("\nThese packages don't seem to be available on PyPI:")
print('{}\n'.format('\n'.join(missing_on_PyPI)))
if to_be_upgraded:
print('These packages need to be upgraded:')
for pkg_upgr in to_be_upgraded:
print(f'{pkg_upgr[0]} ({pkg_upgr[1]} -> {pkg_upgr[2]})')
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Check available versions '
'of the upstream packages on PyPI'
)
parser.add_argument(
'project',
nargs='?',
default='devel:languages:python:numeric',
help='The OpenBuildService project. Defaults ' 'to %(default)s',
)
args = parser.parse_args()
sys.exit(main(args.project))
