diff options
| -rwxr-xr-x | import_issues.py | 173 |
1 files changed, 103 insertions, 70 deletions
diff --git a/import_issues.py b/import_issues.py index 42b0a1f..106c613 100755 --- a/import_issues.py +++ b/import_issues.py @@ -136,7 +136,6 @@ from email.message import EmailMessage from email.utils import format_datetime, make_msgid from pathlib import Path from typing import Any, Optional -from urllib.parse import urlparse logging.basicConfig( format="%(levelname)s:%(funcName)s:%(message)s", @@ -150,8 +149,6 @@ ID_RE = re.compile(r"^[0-9]+$") # to be sure, we will do just 12k MAX_SIZE_COMMENT = 12 * 1024 -tickets_to_be_closed = [] - def split_long_str(in_str: str, max_len: int = MAX_SIZE_COMMENT) -> list[str]: out = [] @@ -167,6 +164,9 @@ def split_long_str(in_str: str, max_len: int = MAX_SIZE_COMMENT) -> list[str]: return out +label_cache: dict[str, list[dict[str, str]]] = {} + + def get_labels(tracker: str) -> list[dict[str, str]]: """ collects labels for your named tracker @@ -174,26 +174,36 @@ def get_labels(tracker: str) -> list[dict[str, str]]: param: tracker: name of the tracker return: list of all labels in the tracker """ - query = ( - 'query { me { tracker(name: "' - + tracker - + '") { labels { results { id, name, foregroundColor, backgroundColor, created } } } }}' - ) + global label_cache - try: - ret = subprocess.run( - ["hut", "graphql", "todo", "--stdin"], - input=query, - text=True, - check=True, - capture_output=True, + result: list[dict[str, str]] + + if tracker in label_cache: + result = label_cache[tracker] + else: + query = ( + 'query { me { tracker(name: "' + + tracker + + '") { labels { results { id, name, foregroundColor, backgroundColor, created } } } }}' ) - except subprocess.CalledProcessError as ex: - raise RuntimeError( - f"hut failed with excitcode {ex.returncode}\n\nstdout:\n{ex.stdout}\n\nand stderr:\n{ex.stderr}" - ) from ex - data = json.loads(ret.stdout) - return data["me"]["tracker"]["labels"]["results"] + + try: + ret = subprocess.run( + ["hut", "graphql", "todo", "--stdin"], + input=query, + text=True, + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError as ex: + raise RuntimeError( + f"hut failed with exitcode {ex.returncode}\n\nstdout:\n{ex.stdout}\n\nand stderr:\n{ex.stderr}" + ) from ex + data = json.loads(ret.stdout) + result = data["me"]["tracker"]["labels"]["results"] + label_cache[tracker] = result + + return result log = logging.getLogger() @@ -284,7 +294,9 @@ def do_mail( raise RuntimeError(f"Unknown mode: {mode!r}") -def run_hut(cmds, tracker, msg, args=None, delay=None): +def run_hut( + cmds, tracker, msg, args=[], *, delay: float +) -> subprocess.CompletedProcess: log.debug( "cmds = %s, tracker = %s, args = %s\n\nmsg:\n%s", cmds, tracker, args, msg ) @@ -310,7 +322,7 @@ def run_hut(cmds, tracker, msg, args=None, delay=None): ) except subprocess.CalledProcessError as ex: raise RuntimeError( - f"hut failed with excitcode {ex.returncode}\n\nstdout:\n{ex.stdout}\n\nand stderr:\n{ex.stderr}" + f"hut failed with exitcode {ex.returncode}\n\nstdout:\n{ex.stdout}\n\nand stderr:\n{ex.stderr}" ) from ex time.sleep(delay) @@ -319,6 +331,7 @@ def run_hut(cmds, tracker, msg, args=None, delay=None): def open_ticket_by_email( + *, smtp, delay: float, mode: str, @@ -335,6 +348,15 @@ def open_ticket_by_email( milestone_name: Optional[str], gitlab_ticket_url: str, ): + # This is presently the ID of the issue we are going to create, following + # along with e.g. any skipped confidential issues. Unfortunately, unlike + # calling 'hut', we don't get a response after sending the email to create + # the ticket that tells us what the ticket ID actually is, so we have to + # assume it's the next one in sequence. It should be, unless ticket + # creation fails for some reason (again, we don't know if that happens, we + # can't see email responses). + global issue_count + lines = [] pheaders = [] @@ -385,24 +407,21 @@ def open_ticket_by_email( # then add remaining parts of the description as comments if len(body_split) > 1: - url_split = urlparse(gitlab_ticket_url) - issue_id = int(os.path.basename(url_split.path)) - for part in body_split[1:]: do_mail( smtp=smtp, delay=delay, mode=mode, frm=frm, - to=f"{tracker}/{issue_id}@todo.sr.ht", + to=f"{tracker}/{issue_count}@todo.sr.ht", body=part, ) def open_ticket_by_hut( + *, delay: float, tracker: str, - frm: str, title: str, body: str, created_by: Optional[str], @@ -413,8 +432,7 @@ def open_ticket_by_hut( labels: list[dict[str, Any]], milestone_name: Optional[str], gitlab_ticket_url: str, -) -> str: - +): lines = [] pheaders = [] @@ -445,7 +463,12 @@ def open_ticket_by_hut( msg_split = split_long_str(msg) out = run_hut(["ticket", "create"], tracker, msg_split[0], delay=delay) - out.issue_id = int(out.stderr.strip()[len("Created new ticket #") :]) + stderr_msg = out.stderr.strip() + expected_prefix = "Created new ticket #" + assert stderr_msg.startswith( + expected_prefix + ), f"Expected stderr to start with {expected_prefix!r}, stderr = {stderr_msg!r}" + issue_id = int(stderr_msg[len(expected_prefix) :]) for label in sorted(labels, key=lambda x: x["label"]["title"]): # {"target_type":"Issue", @@ -468,18 +491,14 @@ def open_ticket_by_hut( ["ticket", "label"], tracker, None, - args=[str(out.issue_id), "-l", label_name], + [str(issue_id), "-l", label_name], delay=delay, ) # then add remaining parts of the description as comments if len(msg_split) > 1: for part in msg_split[1:]: - run_hut( - ["ticket", "comment"], tracker, part, [str(out.issue_id)], delay=delay - ) - - return out + run_hut(["ticket", "comment"], tracker, part, [str(issue_id)], delay=delay) def open_ticket( @@ -502,42 +521,44 @@ def open_ticket( ) -> int: global issue_count + # Proactively increment this so that open_ticket_by_email() can use it for + # the current issue ID. + issue_count += 1 + if mode in ("send", "print"): open_ticket_by_email( - smtp, - delay, - mode, - tracker, - frm, - title, - body, - created_by, - created_at, - closed_at, - is_closed, - is_confidential, - labels, - milestone_name, - gitlab_ticket_url, + smtp=smtp, + delay=delay, + mode=mode, + tracker=tracker, + frm=frm, + title=title, + body=body, + created_by=created_by, + created_at=created_at, + closed_at=closed_at, + is_closed=is_closed, + is_confidential=is_confidential, + labels=labels, + milestone_name=milestone_name, + gitlab_ticket_url=gitlab_ticket_url, ) elif mode == "hut": open_ticket_by_hut( - delay, - tracker, - frm, - title, - body, - created_by, - created_at, - closed_at, - is_closed, - is_confidential, - labels, - milestone_name, - gitlab_ticket_url, + delay=delay, + tracker=tracker, + title=title, + body=body, + created_by=created_by, + created_at=created_at, + closed_at=closed_at, + is_closed=is_closed, + is_confidential=is_confidential, + labels=labels, + milestone_name=milestone_name, + gitlab_ticket_url=gitlab_ticket_url, ) - issue_count += 1 return issue_count @@ -549,6 +570,7 @@ def file_missing_ticket( tracker: str, frm: str, issue_id: int, + tickets_to_be_closed: list[tuple[Any, float, str, str, str, int]], ): global issue_count @@ -680,10 +702,17 @@ def close_ticket( def ensure_label( - tracker: str, name: str, bg_color: str, fg_color: str = "#FFFFFF", delay=None + tracker: str, + name: str, + bg_color: str, + fg_color: str = "#FFFFFF", + *, + delay: float, ): + global label_cache + labels = get_labels(tracker.split("/", 1)[1]) - if not ([x for x in labels if x["name"] == name]): + if not any(x["name"] == name for x in labels): run_hut( ["label", "create"], tracker, @@ -692,6 +721,8 @@ def ensure_label( delay=delay, ) + label_cache.pop(tracker, None) # The label cache is now invalid. + def run( *, @@ -790,6 +821,7 @@ def run( log.info("Creating tickets.") issue_id_map: dict[int, int] = {} + tickets_to_be_closed: list[tuple[Any, float, str, str, str, int]] = [] # While we're creating tickets, we can't just loop over the sorted # issue_jsons. We have to loop over potential issue IDs and handle any that @@ -804,6 +836,7 @@ def run( tracker=tracker, frm=frm, issue_id=gitlab_issue_id, + tickets_to_be_closed=tickets_to_be_closed, ) elif not skip_missing_issues: raise RuntimeError( @@ -926,7 +959,7 @@ def run( is_closed=(issue_json["state"] == "closed"), ) - log.info("Delayed closing issues.") + log.info("Performing delayed closing of issues.") for ticket in tickets_to_be_closed: close_ticket( smtp=ticket[0], @@ -936,7 +969,7 @@ def run( frm=ticket[4], issue_id=ticket[5], closed_at=None, - is_closed=False, + is_closed=False, # Save one line of text. ) |