Using custom hand SPEC file parsing makes script more working.

1 files changed, 78 insertions, 80 deletions

diff --git a/dlpcvp.py b/dlpcvp.py
index f8ccec0..66b60cb 100755
--- a/dlpcvp.py
+++ b/dlpcvp.py
@@ -9,7 +9,6 @@ import os
 import os.path
 import sqlite3
 import sys
-import tempfile
 import urllib.request
 from distutils.version import LooseVersion
 from urllib.error import HTTPError
@@ -17,7 +16,6 @@ from urllib.request import Request, urlopen
 from typing import Iterable, Optional, Tuple
 import xml.etree.ElementTree as ET
 
-import rpm
 
 # PyPI API documentation https://warehouse.readthedocs.io/api-reference/
 PyPI_base = "https://pypi.org/pypi/{}/json"
@@ -79,28 +77,6 @@ def suse_packages(proj: str) -> Iterable[str]:
             raise
 
 
-def parse_spec_in_dev_null(spec_file: str, pkg: str) -> Optional[Tuple[str, str]]:
-    # rpm library generates awful lot of nonsensical goo on
-    # stderr
-    with open(os.devnull, 'wb') as nullf:
-        old_stderr = sys.stderr
-        old_stdout = sys.stdout
-        sys.stderr = nullf
-        sys.stdout = nullf
-        try:
-            spc = rpm.spec(spec_file)
-        except Exception:
-            log.error("Cannot parse {}".format(pkg))
-        else:
-            try:
-                return pkg, spc.packages[0].header['Version'].decode()
-            except IndexError:
-                pass
-        finally:
-            sys.stderr = old_stderr
-            sys.stdout = old_stdout
-
-
 def get_etags(con: sqlite3.Connection, pkg: str) -> \
         Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
     # pkg, suse_name_etag, suse_spec_etag, pypi_etag
@@ -128,13 +104,60 @@ def update_etags(con: sqlite3.Connection, pkg: str, e_suse_name: Optional[str],
     con.commit()
 
 
-def get_version_from_pypi(name: str, con: sqlite3.Connection) \
-        -> Optional[Tuple[str, str]]:
+def parse_spec(spec_file: str, pkg: str) -> LooseVersion:
+    # rpm library generates awful lot of nonsensical goo on stderr
+    if isinstance(spec_file, bytes):
+        spec_file = spec_file.decode()
+    for line in spec_file.split('\n'):
+        if line.startswith('Version:'):
+            rest_of_line = line[len('Version:'):].strip()
+
+    return LooseVersion(rest_of_line)
+
+
+def get_spec_name(req: Request, pkg: str, etag: str = None) -> LooseVersion:
+    # Acquire version from the listing of the project directory
+    spec_files = []
+
+    if etag is not None:
+        req.add_header('ETag', etag)
+
+    if not spec_files:
+        IOError(f'Cannot find SPEC file for {pkg}.')
+
+    try:
+        with opener.open(req) as resp:
+            etag = resp.info()['ETag']
+            raw_xml_data = ET.parse(resp)
+            root = raw_xml_data.getroot()
+            for elem in root.iter('entry'):
+                name = elem.get('name')
+                if name.endswith('.spec'):
+                    spec_files.append(name)
+    except HTTPError as ex:
+        if ex.getcode() == 404:
+            log.error(f'Cannot acquire version of {pkg}.')
+            return None
+        else:
+            raise
+
+    try:
+        fname = sorted(spec_files, key=len)[0]
+        return fname
+    except IndexError:
+        log.exception(f'Cannot find correct spec_files: {spec_files} ({type(spec_files)}) for {pkg}')
+        return
+
+
+def get_version_from_pypi(name: str, con: sqlite3.Connection = None) -> Optional[Tuple[str, str]]:
     """
     For the given name of module return the latest version available on PyPI.
     """
     # pkg, suse_name_etag, suse_spec_etag, pypi_etag
-    _, _, _, etag = get_etags(con, name)
+    if con:
+        _, _, _, etag = get_etags(con, name)
+    else:
+        etag = None
     req = Request(url=PyPI_base.format(name))
 
     if etag is not None:
@@ -144,8 +167,9 @@ def get_version_from_pypi(name: str, con: sqlite3.Connection) \
         with urlopen(req) as resp:
             data = json.load(resp)
             info_dict = data['info']
-            update_etags(con, name, None, None, resp.info()['ETag'])
-            return info_dict['name'], info_dict['version']
+            if con:
+                update_etags(con, name, None, None, resp.info()['ETag'])
+            return LooseVersion(info_dict['version'])
     except HTTPError as ex:
         if ex.getcode() == 404:
             log.error(f'Cannot find {name} on PyPI')
@@ -154,7 +178,7 @@ def get_version_from_pypi(name: str, con: sqlite3.Connection) \
             raise
 
 
-def package_version(proj: str, pkg_name: str, con: sqlite3.Connection) \
+def package_version(proj: str, pkg_name: str, con: sqlite3.Connection = None) \
                     -> Optional[Tuple[str, str, str]]:
     """
     Return the version of the given package in the given proj.
@@ -162,38 +186,14 @@ def package_version(proj: str, pkg_name: str, con: sqlite3.Connection) \
     Downloads SPEC file from OBS and parses it.
     """
     # pkg, suse_name_etag, suse_spec_etag, pypi_etag
-    _, etag_fn, etag_spcf, _ = get_etags(con, pkg_name)
+    if con:
+        _, etag_fn, etag_spcf, _ = get_etags(con, pkg_name)
+    else:
+        etag_fn, etag_spcf = None, None
 
+    # Get listing of the package repository
     req_spc_name = Request(url=OBS_base + f'/source/{proj}/{pkg_name}')
-    spec_files = []
-
-    if etag_fn is not None:
-        req_spc_name.add_header('ETag', etag_fn)
-
-    try:
-        with opener.open(req_spc_name) as resp:
-            etag_fn = resp.info()['ETag']
-            raw_xml_data = ET.parse(resp)
-            root = raw_xml_data.getroot()
-            for elem in root.iter('entry'):
-                name = elem.get('name')
-                if name.endswith('.spec'):
-                    spec_files.append(name)
-    except HTTPError as ex:
-        if ex.getcode() == 404:
-            log.error(f'Cannot acquire version of {pkg_name} in {proj}')
-            return None
-        else:
-            raise
-
-    if not spec_files:
-        IOError(f'Cannot find SPEC file for {pkg_name}')
-
-    try:
-        spc_fname = sorted(spec_files, key=len)[0]
-    except IndexError:
-        log.exception(f'Cannot find correct spec_files: {spec_files}')
-        return
+    spc_fname = get_spec_name(req_spc_name, pkg_name, etag_fn)
 
     req_spec = Request(url=OBS_base + f'/source/{proj}/{pkg_name}/{spc_fname}')
 
@@ -204,15 +204,10 @@ def package_version(proj: str, pkg_name: str, con: sqlite3.Connection) \
         with opener.open(req_spec) as resp:
             etag_spcf = resp.info()['ETag']
             spec_file_str = resp.read()
-            spec_file_name = ''
-            with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as spf:
-                spec_file_name = spf.name
-                spf.write(spec_file_str)
-                spf.flush()
-                os.fsync(spf.fileno())
 
+            if con:
                 update_etags(con, pkg_name, etag_fn, etag_spcf, None)
-                return parse_spec_in_dev_null(spec_file_name, pkg_name)
+            return parse_spec(spec_file_str, pkg_name)
     except HTTPError as ex:
         if ex.getcode() == 404:
             log.error(f'Cannot parse SPEC file {spc_fname} for {pkg_name}')
@@ -232,24 +227,24 @@ def main(prj):
             if pkg.startswith('python-'):
                 pypi_name = pkg[CUTCHARS:]
                 try:
-                    suse_ver_tup = package_version(prj, pkg, conn)
-                    if suse_ver_tup is None:
+                    suse_ver = package_version(prj, pkg, conn)
+                    if suse_ver is None:
                         raise RuntimeError('not in OBS')
-                    pypi_ver_tup = get_version_from_pypi(pypi_name, conn)
-                    if pypi_ver_tup is None:
+                    pypi_ver = get_version_from_pypi(pypi_name, conn)
+                    if pypi_ver is None:
                         raise RuntimeError('not in PyPI')
                 except RuntimeError as ex:
                     log.warning(f'Package {pkg} cannot be found: {ex}')
                     continue
 
-                log.debug(f"{pkg} {suse_ver_tup} {pypi_ver_tup}")
-
-                pypi_ver = LooseVersion(pypi_ver_tup[1])
-                suse_ver = LooseVersion(suse_ver_tup[1])
-
-                if pypi_ver > suse_ver:
-                    print(
-                        f'We need to upgrade {pkg} ({suse_ver} -> {pypi_ver})')
+                try:
+                    if pypi_ver > suse_ver:
+                        print(
+                            f'We need to upgrade {pkg} ({suse_ver} -> {pypi_ver})')
+                except TypeError:
+                    log.error(f'{pkg} pypi_ver = {pypi_ver}')
+                    log.error(f'{pkg} suse_ver = {suse_ver}')
+                    continue
             else:
                 print(f'Is {pkg} on PyPI?')
 
@@ -258,9 +253,12 @@ if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Check available versions '
                                      'of the upstream packages on PyPI')
     parser.add_argument('--opensuse-project',
-                        default='devel:languages:python',
+                        default='devel:languages:python:numeric',
                         help='The OpenBuildService project. Defaults '
                         'to %(default)s')
 
     args = parser.parse_args()
     sys.exit(main(args.opensuse_project))
+
+
+