Finally, hopefully, version caching ETags to SQLite.

1 files changed, 52 insertions, 46 deletions

diff --git a/dlpcvp.py b/dlpcvp.py
index 8e117ab..1a54b92 100755
--- a/dlpcvp.py
+++ b/dlpcvp.py
@@ -31,13 +31,16 @@ config = configparser.ConfigParser()
 config.read(ConfigRC)
 
 logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s',
-                    stream=sys.stdout, level=logging.DEBUG)
+                    stream=sys.stdout, level=logging.INFO)
 log = logging.getLogger()
 
 # or HTTPPasswordMgrWithPriorAuth ?
 password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
 user = config[OBS_base]['user']
 passw = config[OBS_base]['pass']
+# MYPY: Argument 1 to "add_password" of
+# "HTTPPasswordMgrWithDefaultRealm" has incompatible type "None";
+# expected "str"
 password_mgr.add_password(None, OBS_base, user, passw)
 
 handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
@@ -54,28 +57,6 @@ TB_CREATE = """CREATE TABLE etags
 TB_CREATE_IDX = "CREATE UNIQUE INDEX etags_names ON etags(pkg)"
 
 
-def get_version_from_pypi(name: str, etag: str = None) -> Tuple[str, str, str]:
-    """
-    For the given name of module return the latest version available on PyPI.
-    """
-    req = Request(url=PyPI_base.format(name))
-
-    if etag is not None:
-        req.add_header('ETag', etag)
-
-    try:
-        with urlopen(req) as resp:
-            data = json.load(resp)
-            info_dict = data['info']
-            return info_dict['name'], info_dict['version'], resp.info()['ETag']
-    except HTTPError as ex:
-        if ex.getcode() == 404:
-            log.error(f'Cannot find {name} on PyPI')
-            return None
-        else:
-            raise
-
-
 def suse_packages(proj: str) -> Iterable[str]:
     """
     Iterator returning names of all packages in the given proj
@@ -98,7 +79,7 @@ def suse_packages(proj: str) -> Iterable[str]:
             raise
 
 
-def parse_spec_in_dev_null(spec_file, pkg, e_fn, e_spcf):
+def parse_spec_in_dev_null(spec_file, pkg):
     # rpm library generates awful lot of nonsensical goo on
     # stderr
     with open(os.devnull, 'wb') as nullf:
@@ -112,8 +93,7 @@ def parse_spec_in_dev_null(spec_file, pkg, e_fn, e_spcf):
             log.error("Cannot parse {}".format(pkg))
         else:
             try:
-                return pkg, spc.packages[0].header['Version'].decode(), \
-                       e_fn, e_spcf
+                return pkg, spc.packages[0].header['Version'].decode()
             except IndexError:
                 pass
         finally:
@@ -121,7 +101,9 @@ def parse_spec_in_dev_null(spec_file, pkg, e_fn, e_spcf):
             sys.stdout = old_stdout
 
 
-def get_etags(con, pkg):
+def get_etags(con: sqlite3.Connection, pkg: str) -> \
+        Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
+    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
     cur = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,))
     ret = cur.fetchone()
     if ret is None:
@@ -130,8 +112,12 @@ def get_etags(con, pkg):
         return ret
 
 
-def update_etags(con, pkg, e_suse_name, e_suse_spec, e_pypi):
-    if con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone():
+def update_etags(con: sqlite3.Connection, pkg: str, e_suse_name: Optional[str], e_suse_spec: Optional[str], e_pypi: Optional[str]):
+    res = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone()
+    if res:
+        e_suse_name = res[1] if e_suse_name is None else e_suse_name
+        e_suse_spec = res[2] if e_suse_spec is None else e_suse_spec
+        e_pypi = res[3] if e_pypi is None else e_pypi
         con.execute('''UPDATE etags
             SET suse_name_etag=?, suse_spec_etag=?, pypi_etag=?
             WHERE pkg=?''', (e_suse_name, e_suse_spec, e_pypi, pkg))
@@ -142,14 +128,42 @@ def update_etags(con, pkg, e_suse_name, e_suse_spec, e_pypi):
     con.commit()
 
 
-def package_version(proj: str, pkg_name: str,
-                    etag_fn: str = None, etag_spcf: str = None) \
-                        -> Optional[Tuple[str, str, str]]:
+def get_version_from_pypi(name: str, con: sqlite3.Connection) \
+        -> Optional[Tuple[str, str]]:
+    """
+    For the given name of module return the latest version available on PyPI.
+    """
+    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
+    _, _, _, etag = get_etags(con, name)
+    req = Request(url=PyPI_base.format(name))
+
+    if etag is not None:
+        req.add_header('ETag', etag)
+
+    try:
+        with urlopen(req) as resp:
+            data = json.load(resp)
+            info_dict = data['info']
+            update_etags(con, name, None, None, resp.info()['ETag'])
+            return info_dict['name'], info_dict['version']
+    except HTTPError as ex:
+        if ex.getcode() == 404:
+            log.error(f'Cannot find {name} on PyPI')
+            return None
+        else:
+            raise
+
+
+def package_version(proj: str, pkg_name: str, con: sqlite3.Connection) \
+                    -> Optional[Tuple[str, str, str]]:
     """
     Return the version of the given package in the given proj.
 
     Downloads SPEC file from OBS and parses it.
     """
+    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
+    _, etag_fn, etag_spcf, _ = get_etags(con, pkg_name)
+
     req_spc_name = Request(url=OBS_base + f'/source/{proj}/{pkg_name}')
     spec_files = []
 
@@ -168,7 +182,7 @@ def package_version(proj: str, pkg_name: str,
     except HTTPError as ex:
         if ex.getcode() == 404:
             log.error(f'Cannot accquire version of {pkg_name} in {proj}')
-            return
+            return None
         else:
             raise
 
@@ -197,8 +211,8 @@ def package_version(proj: str, pkg_name: str,
                 spf.flush()
                 os.fsync(spf.fileno())
 
-                return parse_spec_in_dev_null(spec_file_name, pkg_name,
-                                              etag_fn, etag_spcf)
+                update_etags(con, pkg_name, etag_fn, etag_spcf, None)
+                return parse_spec_in_dev_null(spec_file_name, pkg_name)
     except HTTPError as ex:
         if ex.getcode() == 404:
             log.error(f'Cannot parse SPEC file {spc_fname} for {pkg_name}')
@@ -217,21 +231,13 @@ def main(prj):
             log.debug(f'pkg = {pkg}')
             if pkg.startswith('python-'):
                 pypi_name = pkg[CUTCHARS:]
-                _, et_suse_name, et_suse_spec, et_pypi = get_etags(conn, pkg)
-
                 try:
-                    suse_ver_tup = package_version(prj, pkg,
-                                                   et_suse_name, et_suse_spec)
+                    suse_ver_tup = package_version(prj, pkg, conn)
                     if suse_ver_tup is None:
                         raise RuntimeError('not in OBS')
-
-                    pypi_ver_tup = get_version_from_pypi(pypi_name, et_pypi)
+                    pypi_ver_tup = get_version_from_pypi(pypi_name, conn)
                     if pypi_ver_tup is None:
                         raise RuntimeError('not in PyPI')
-
-                    update_etags(conn, pkg,
-                                 suse_ver_tup[1], suse_ver_tup[2],
-                                 pypi_ver_tup[2])
                 except RuntimeError as ex:
                     log.warning(f'Package {pkg} cannot be found: {ex}')
                     continue
@@ -252,7 +258,7 @@ if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Check available versions '
                                      'of the upstream packages on PyPI')
     parser.add_argument('--opensuse-project',
-                        default='devel:languages:python:numeric',
+                        default='devel:languages:python',
                         help='The OpenBuildService project. Defaults '
                         'to %(default)s')