#!/usr/bin/python3
# Requires: python3-rpm
import argparse
import configparser
import json
import logging
import os
import os.path
import sqlite3
import sys
import urllib.request
from distutils.version import LooseVersion
from urllib.error import HTTPError
from urllib.request import Request, urlopen
from typing import Iterable, Optional, Tuple
import xml.etree.ElementTree as ET
# PyPI API documentation https://warehouse.readthedocs.io/api-reference/
PyPI_base = "https://pypi.org/pypi/{}/json"
# https://github.com/openSUSE/open-build-service/blob/master/docs/api/api/api.txt
# https://build.opensuse.org/apidocs/index
OBS_base = "https://api.opensuse.org"
ConfigRC = os.path.expanduser('~/.config/osc/oscrc')
CUTCHARS = len('python-')
config = configparser.ConfigParser()
config.read(ConfigRC)
logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s',
stream=sys.stdout, level=logging.INFO)
log = logging.getLogger()
# or HTTPPasswordMgrWithPriorAuth ?
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
user = config[OBS_base]['user']
passw = config[OBS_base]['pass']
# MYPY: Argument 1 to "add_password" of
# "HTTPPasswordMgrWithDefaultRealm" has incompatible type "None";
# expected "str"
password_mgr.add_password(None, OBS_base, user, passw)
handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
opener = urllib.request.build_opener(handler)
urllib.request.install_opener(opener)
TB_EXISTS = """SELECT name FROM sqlite_master
WHERE type='table' AND name='etags'
"""
TB_CREATE = """CREATE TABLE etags
(pkg text NOT NULL,
suse_name_etag text, suse_spec_etag text, pypi_etag text)
"""
TB_CREATE_IDX = "CREATE UNIQUE INDEX etags_names ON etags(pkg)"
def suse_packages(proj: str) -> Iterable[str]:
"""
Iterator returning names of all packages in the given proj
ETag management won't work here, because I don't know about any way
how to return it in iterator.
"""
req = Request(url=OBS_base + f'/source/{proj}')
try:
with opener.open(req) as resp:
raw_xml_data = ET.parse(resp)
root = raw_xml_data.getroot()
for elem in root.iter('entry'):
yield elem.get('name')
except HTTPError as ex:
if ex.getcode() == 404:
log.error(f'Cannot find packages for {proj}!')
else:
raise
def get_etags(con: sqlite3.Connection, pkg: str) -> \
Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
# pkg, suse_name_etag, suse_spec_etag, pypi_etag
cur = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,))
ret = cur.fetchone()
if ret is None:
return None, None, None, None
else:
return ret
def update_etags(con: sqlite3.Connection, pkg: str, e_suse_name: Optional[str], e_suse_spec: Optional[str], e_pypi: Optional[str]):
res = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone()
if res:
e_suse_name = res[1] if e_suse_name is None else e_suse_name
e_suse_spec = res[2] if e_suse_spec is None else e_suse_spec
e_pypi = res[3] if e_pypi is None else e_pypi
con.execute('''UPDATE etags
SET suse_name_etag=?, suse_spec_etag=?, pypi_etag=?
WHERE pkg=?''', (e_suse_name, e_suse_spec, e_pypi, pkg))
else:
con.execute('''INSERT INTO etags
(pkg, suse_name_etag, suse_spec_etag, pypi_etag)
VALUES (?, ?, ?, ?)''', (pkg, e_suse_name, e_suse_spec, e_pypi))
con.commit()
def parse_spec(spec_file: str, pkg: str) -> LooseVersion:
# rpm library generates awful lot of nonsensical goo on stderr
if isinstance(spec_file, bytes):
spec_file = spec_file.decode()
rest_of_line = ''
for line in spec_file.split('\n'):
if line.startswith('Version:'):
rest_of_line = line[len('Version:'):].strip()
break
return LooseVersion(rest_of_line)
def get_spec_name(req: Request, pkg: str, etag: str = None) -> Optional[str]:
# Acquire version from the listing of the project directory
spec_files = []
if etag is not None:
req.add_header('ETag', etag)
if not spec_files:
IOError(f'Cannot find SPEC file for {pkg}.')
try:
with opener.open(req) as resp:
etag = resp.info()['ETag']
raw_xml_data = ET.parse(resp)
root = raw_xml_data.getroot()
for elem in root.iter('entry'):
name = elem.get('name')
if name.endswith('.spec'):
spec_files.append(name)
except HTTPError as ex:
if ex.getcode() == 404:
log.error(f'Cannot acquire version of {pkg}.')
return None
else:
raise
try:
fname = sorted(spec_files, key=len)[0]
return fname
except IndexError:
log.exception(f'Cannot find correct spec_files: {spec_files} ({type(spec_files)}) for {pkg}')
return
def get_version_from_pypi(name: str, con: sqlite3.Connection = None) -> Optional[LooseVersion]:
"""
For the given name of module return the latest version available on PyPI.
"""
# pkg, suse_name_etag, suse_spec_etag, pypi_etag
if con:
_, _, _, etag = get_etags(con, name)
else:
etag = None
req = Request(url=PyPI_base.format(name))
if etag is not None:
req.add_header('ETag', etag)
try:
with urlopen(req) as resp:
data = json.load(resp)
info_dict = data['info']
if con:
update_etags(con, name, None, None, resp.info()['ETag'])
return LooseVersion(info_dict['version'])
except HTTPError as ex:
if ex.getcode() == 404:
log.error(f'Cannot find {name} on PyPI')
return None
else:
raise
def package_version(proj: str, pkg_name: str, con: sqlite3.Connection = None) -> LooseVersion:
"""
Return the version of the given package in the given proj.
Downloads SPEC file from OBS and parses it.
"""
# pkg, suse_name_etag, suse_spec_etag, pypi_etag
if con:
_, etag_fn, etag_spcf, _ = get_etags(con, pkg_name)
else:
etag_fn, etag_spcf = None, None
# Get listing of the package repository
req_spc_name = Request(url=OBS_base + f'/source/{proj}/{pkg_name}')
spc_fname = get_spec_name(req_spc_name, pkg_name, etag_fn)
req_spec = Request(url=OBS_base + f'/source/{proj}/{pkg_name}/{spc_fname}')
if etag_spcf is not None:
req_spc_name.add_header('ETag', etag_spcf)
try:
with opener.open(req_spec) as resp:
etag_spcf = resp.info()['ETag']
spec_file_str = resp.read()
if con:
update_etags(con, pkg_name, etag_fn, etag_spcf, None)
return parse_spec(spec_file_str, pkg_name)
except HTTPError as ex:
if ex.getcode() == 404:
log.error(f'Cannot parse SPEC file {spc_fname} for {pkg_name}')
else:
raise
def main(prj):
db_name = os.path.split(os.path.basename(sys.argv[0]))[0] + ".db"
with sqlite3.connect(db_name) as conn:
if not conn.execute(TB_EXISTS).fetchone():
conn.execute(TB_CREATE)
conn.execute(TB_CREATE_IDX)
for pkg in suse_packages(prj):
log.debug(f'pkg = {pkg}')
if pkg.startswith('python-'):
pypi_name = pkg[CUTCHARS:]
try:
suse_ver = package_version(prj, pkg, conn)
if suse_ver is None:
raise RuntimeError('not in OBS')
pypi_ver = get_version_from_pypi(pypi_name, conn)
if pypi_ver is None:
raise RuntimeError('not in PyPI')
except RuntimeError as ex:
log.warning(f'Package {pkg} cannot be found: {ex}')
continue
try:
if pypi_ver > suse_ver:
print(
f'We need to upgrade {pkg} ({suse_ver} -> {pypi_ver})')
except TypeError:
log.error(f'{pkg} pypi_ver = {pypi_ver}')
log.error(f'{pkg} suse_ver = {suse_ver}')
continue
else:
print(f'Is {pkg} on PyPI?')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Check available versions '
'of the upstream packages on PyPI')
parser.add_argument('--opensuse-project',
default='devel:languages:python:numeric',
help='The OpenBuildService project. Defaults '
'to %(default)s')
args = parser.parse_args()
sys.exit(main(args.opensuse_project))
