diff options
| -rwxr-xr-x | import_issues.py | 477 |
1 files changed, 248 insertions, 229 deletions
diff --git a/import_issues.py b/import_issues.py index 666644e..d301c1d 100755 --- a/import_issues.py +++ b/import_issues.py @@ -136,10 +136,12 @@ from pathlib import Path from typing import Dict, List, Optional -ID_RE = re.compile(r'^[0-9]+$') +ID_RE = re.compile(r"^[0-9]+$") -logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s', - level=logging.DEBUG) +logging.basicConfig( + format="%(levelname)s:%(funcName)s:%(message)s", + level=logging.DEBUG, +) log = logging.getLogger() email_count = 0 @@ -150,30 +152,32 @@ def read_id_map_file(file_path: Path) -> Dict[int, str]: """Reads a CSV file with ID,NAME mappings and returns the resulting dict.""" result: Dict[int, str] = {} - with open(file_path, newline='') as fh: + with open(file_path, newline="") as fh: reader = csv.reader(fh) line_num = 0 for row in reader: line_num += 1 - assert len(row) == 2 and ID_RE.search(row[0]) and row[1], \ - f"Row {line_num} of {file_path} is not in the form <ID>,<NAME>: {row!r}" + assert ( + len(row) == 2 and ID_RE.search(row[0]) and row[1] + ), f"Row {line_num} of {file_path} is not in the form <ID>,<NAME>: {row!r}" new_id = int(row[0]) - assert new_id not in result, \ - f"ID {new_id} appears multiple times in {file_path}." + assert ( + new_id not in result + ), f"ID {new_id} appears multiple times in {file_path}." result[new_id] = row[1] return result def do_mail( - *, - smtp, - smtp_delay: float, - mode: str, - frm: str, - to: str, - body: str, - subject: Optional[str] = None, + *, + smtp, + smtp_delay: float, + mode: str, + frm: str, + to: str, + body: str, + subject: Optional[str] = None, ): global email_count email_count += 1 @@ -217,23 +221,23 @@ def do_mail( def open_ticket( - *, - smtp, - smtp_delay: float, - mode: str, - srht_owner: str, - srht_tracker: str, - frm: str, - title: str, - body: str, - created_by: Optional[str], - created_at: str, - closed_at: Optional[str], - is_closed: bool, - is_confidential: bool, - label_names: List[str], - milestone_name: Optional[str], - gitlab_ticket_url: str, + *, + smtp, + smtp_delay: float, + mode: str, + srht_owner: str, + srht_tracker: str, + frm: str, + title: str, + body: str, + created_by: Optional[str], + created_at: str, + closed_at: Optional[str], + is_closed: bool, + is_confidential: bool, + label_names: List[str], + milestone_name: Optional[str], + gitlab_ticket_url: str, ) -> int: global issue_count @@ -279,14 +283,14 @@ def open_ticket( def file_missing_ticket( - *, - smtp, - smtp_delay: float, - mode: str, - srht_owner: str, - srht_tracker: str, - frm: str, - issue_id: int, + *, + smtp, + smtp_delay: float, + mode: str, + srht_owner: str, + srht_tracker: str, + frm: str, + issue_id: int, ): global issue_count @@ -318,20 +322,20 @@ def file_missing_ticket( def send_comment( - *, - smtp, - smtp_delay: float, - mode: str, - srht_owner: str, - srht_tracker: str, - frm: str, - issue_id: int, - body: str, - author_name: str, - created_at: str, - last_edited_at: str, - is_system: bool, - is_confidential: bool, + *, + smtp, + smtp_delay: float, + mode: str, + srht_owner: str, + srht_tracker: str, + frm: str, + issue_id: int, + body: str, + author_name: str, + created_at: str, + last_edited_at: str, + is_system: bool, + is_confidential: bool, ): lines = [] pheaders = [] @@ -368,16 +372,16 @@ def send_comment( def close_ticket( - *, - smtp, - smtp_delay: float, - mode: str, - srht_owner: str, - srht_tracker: str, - frm: str, - issue_id: int, - closed_at: Optional[str], - is_closed: bool, + *, + smtp, + smtp_delay: float, + mode: str, + srht_owner: str, + srht_tracker: str, + frm: str, + issue_id: int, + closed_at: Optional[str], + is_closed: bool, ): lines = [] @@ -401,84 +405,81 @@ def close_ticket( def run( - *, - smtp, - smtp_delay: float, - mode: str, - srht_owner: str, - srht_tracker: str, - frm: str, - export_dir_path: Path, - gitlab_project_url: str, - labels_file_path: Optional[Path], - skip_unknown_labels: bool, - users_file_path: Optional[Path], - skip_unknown_users: bool, - skip_missing_issues: bool, - create_missing_issues: bool, - include_confidential: bool, - skip_confidential: bool, + *, + smtp, + smtp_delay: float, + mode: str, + srht_owner: str, + srht_tracker: str, + frm: str, + export_dir_path: Path, + gitlab_project_url: str, + labels_file_path: Optional[Path], + skip_unknown_labels: bool, + users_file_path: Optional[Path], + skip_unknown_users: bool, + skip_missing_issues: bool, + create_missing_issues: bool, + include_confidential: bool, + skip_confidential: bool, ): - label_ids_to_names: Optional[Dict[int, str]] = \ + label_ids_to_names: Optional[Dict[int, str]] = ( read_id_map_file(labels_file_path) if labels_file_path else None - user_ids_to_names: Optional[Dict[int, str]] = \ + ) + user_ids_to_names: Optional[Dict[int, str]] = ( read_id_map_file(users_file_path) if users_file_path else None + ) # TODO Might be able to automatically map note.events.author_id to # note.author.name for a subset of relevant users. milestone_jsons = [] - with open(export_dir_path / 'milestones.ndjson') as milestones_file: + with open(export_dir_path / "milestones.ndjson") as milestones_file: for line in milestones_file: milestone_jsons.append(json.loads(line)) milestone_ids_to_titles = {} for milestone_json in milestone_jsons: - milestone_ids_to_titles[milestone_json['iid']] = milestone_json['title'] + milestone_ids_to_titles[milestone_json["iid"]] = milestone_json["title"] issue_jsons = [] - with open(export_dir_path / 'issues.ndjson') as issues_file: + with open(export_dir_path / "issues.ndjson") as issues_file: for line in issues_file: issue_jsons.append(json.loads(line)) if skip_confidential: - issue_jsons = [x for x in issue_jsons if not x.get('confidential')] + issue_jsons = [x for x in issue_jsons if not x.get("confidential")] for issue_json in issue_jsons: - issue_json['notes'] = [ - n - for n in issue_json['notes'] - if not n.get('confidential') + issue_json["notes"] = [ + n for n in issue_json["notes"] if not n.get("confidential") ] elif not include_confidential: - have_confidential_issues = any( - x.get('confidential') - for x in issue_jsons - ) + have_confidential_issues = any(x.get("confidential") for x in issue_jsons) have_confidential_notes = any( - n.get('confidential') - for x in issue_jsons - for n in x['notes'] + n.get("confidential") for x in issue_jsons for n in x["notes"] ) confidential_types = [] if have_confidential_issues: - confidential_types.append('issues') + confidential_types.append("issues") if have_confidential_notes: - confidential_types.append('notes') - assert not (have_confidential_issues or have_confidential_notes), \ - f"Found confidential {' and '.join(confidential_types)}; please " \ - f"decide whether these should all be included, then pass either " \ - f"--include-confidential or --skip-confidential, or edit " \ + confidential_types.append("notes") + assert not (have_confidential_issues or have_confidential_notes), ( + f"Found confidential {' and '.join(confidential_types)}; please " + f"decide whether these should all be included, then pass either " + f"--include-confidential or --skip-confidential, or edit " f"issues.ndjson for more fine-grained control." + ) - issue_jsons.sort(key=lambda x: x['iid']) + issue_jsons.sort(key=lambda x: x["iid"]) - max_issue_id = max(x['iid'] for x in issue_jsons) - present_issue_id_set = {x['iid'] for x in issue_jsons} + max_issue_id = max(x["iid"] for x in issue_jsons) + present_issue_id_set = {x["iid"] for x in issue_jsons} missing_issue_ids = set(range(1, max_issue_id + 1)) - present_issue_id_set if missing_issue_ids and not (skip_missing_issues or create_missing_issues): if skip_confidential: - because_confidential_msg = \ + because_confidential_msg = ( " (possibly because some confidential issues were excluded)" + ) else: because_confidential_msg = "" @@ -489,11 +490,11 @@ def run( issues_by_id = {} for issue_json in issue_jsons: - issues_by_id[issue_json['iid']] = issue_json + issues_by_id[issue_json["iid"]] = issue_json # Need to sort notes by date, they seem to come unsorted. for issue_json in issue_jsons: - issue_json['notes'].sort(key=lambda x: x['created_at']) + issue_json["notes"].sort(key=lambda x: x["created_at"]) log.info("-------- CREATING TICKETS") @@ -524,16 +525,17 @@ def run( issue_json = issues_by_id[gitlab_issue_id] - author_id = issue_json['author_id'] + author_id = issue_json["author_id"] created_by: Optional[str] if user_ids_to_names is None: created_by = None elif author_id in user_ids_to_names: created_by = user_ids_to_names[author_id] else: - assert skip_unknown_users, \ - f"Unknown author #{author_id} of ticket #{gitlab_issue_id}, " \ + assert skip_unknown_users, ( + f"Unknown author #{author_id} of ticket #{gitlab_issue_id}, " f"please add to the users file." + ) created_by = None srht_issue_id = open_ticket( @@ -543,57 +545,67 @@ def run( srht_owner=srht_owner, srht_tracker=srht_tracker, frm=frm, - title=issue_json['title'], - body=issue_json['description'], + title=issue_json["title"], + body=issue_json["description"], created_by=created_by, - created_at=issue_json['created_at'], - closed_at=issue_json['closed_at'], - is_closed=(issue_json['state'] == 'closed'), - is_confidential=(issue_json.get('confidential') is True), - label_names=[x['label']['title'] for x in issue_json['label_links']], - milestone_name=issue_json.get('milestone', {}).get('title') or None, + created_at=issue_json["created_at"], + closed_at=issue_json["closed_at"], + is_closed=(issue_json["state"] == "closed"), + is_confidential=(issue_json.get("confidential") is True), + label_names=[x["label"]["title"] for x in issue_json["label_links"]], + milestone_name=issue_json.get("milestone", {}).get("title") or None, gitlab_ticket_url=f"{gitlab_project_url}/-/issues/{gitlab_issue_id}", ) if not skip_missing_issues: - assert srht_issue_id == gitlab_issue_id, \ - f"Internal error, srht_issue_id {srht_issue_id} != " \ - f"gitlab_issue_id {gitlab_issue_id} " \ - f"(skip_missing_issues={skip_missing_issues}, " \ + assert srht_issue_id == gitlab_issue_id, ( + f"Internal error, srht_issue_id {srht_issue_id} != " + f"gitlab_issue_id {gitlab_issue_id} " + f"(skip_missing_issues={skip_missing_issues}, " f"create_missing_issues={create_missing_issues})." + ) issue_id_map[gitlab_issue_id] = srht_issue_id log.info("-------- CREATING COMMENTS") for issue_json in issue_jsons: - for note_json in issue_json['notes']: - system_action = note_json.get('system_note_metadata', {}).get('action', None) + for note_json in issue_json["notes"]: + system_action = note_json.get("system_note_metadata", {}).get( + "action", None + ) - body = note_json['note'] + body = note_json["note"] # The "Removed" part is a guess here, don't know if that actually shows up. if label_ids_to_names is not None and ( - system_action == 'label' or re.search(r'^(Added|Removed) ~[0-9]+ label', body) + system_action == "label" + or re.search(r"^(Added|Removed) ~[0-9]+ label", body) ): + def expand_label(ref): ref_num = int(ref.group(1)) if ref_num in label_ids_to_names: return label_ids_to_names[ref_num] - assert skip_unknown_labels, \ - f"Unknown label #{ref_num}, please add to the labels file." + assert ( + skip_unknown_labels + ), f"Unknown label #{ref_num}, please add to the labels file." return ref.group(0) # Return the original "~id" string. - body = re.sub(r'~([0-9]+)', expand_label, body) + body = re.sub(r"~([0-9]+)", expand_label, body) + + if system_action == "milestone" or re.search( + r"^Milestone changed to %[0-9]+$", body + ): - if system_action == 'milestone' or re.search(r'^Milestone changed to %[0-9]+$', body): def expand_milestone(ref): ref_num = int(ref.group(1)) - assert ref_num in milestone_ids_to_titles, \ - f"Unknown milestone #{ref_num}." + assert ( + ref_num in milestone_ids_to_titles + ), f"Unknown milestone #{ref_num}." return milestone_ids_to_titles[ref_num] - body = re.sub(r'%([0-9]+)', expand_milestone, body) + body = re.sub(r"%([0-9]+)", expand_milestone, body) send_comment( smtp=smtp, @@ -602,19 +614,19 @@ def run( srht_owner=srht_owner, srht_tracker=srht_tracker, frm=frm, - issue_id=issue_id_map[issue_json['iid']], + issue_id=issue_id_map[issue_json["iid"]], body=body, - author_name=note_json['author']['name'], - created_at=note_json['created_at'], - last_edited_at=note_json['last_edited_at'], - is_system=note_json['system'], - is_confidential=(note_json['confidential'] is True), + author_name=note_json["author"]["name"], + created_at=note_json["created_at"], + last_edited_at=note_json["last_edited_at"], + is_system=note_json["system"], + is_confidential=(note_json["confidential"] is True), ) log.info("-------- CLOSING CLOSED ISSUES") for issue_json in issue_jsons: - if issue_json['state'] == 'closed': + if issue_json["state"] == "closed": close_ticket( smtp=smtp, smtp_delay=smtp_delay, @@ -622,191 +634,198 @@ def run( srht_owner=srht_owner, srht_tracker=srht_tracker, frm=frm, - issue_id=issue_id_map[issue_json['iid']], - closed_at=issue_json['closed_at'], - is_closed=(issue_json['state'] == 'closed'), + issue_id=issue_id_map[issue_json["iid"]], + closed_at=issue_json["closed_at"], + is_closed=(issue_json["state"] == "closed"), ) def main(): parser = argparse.ArgumentParser( - prog='import_issues.py', - description='Import Gitlab issues into Sourcehut via SMTP.', + prog="import_issues.py", + description="Import Gitlab issues into Sourcehut via SMTP.", ) parser.add_argument( - '--srht-owner', + "--srht-owner", required=True, - help='Owner of the Sorucehut tracker.', + help="Owner of the Sorucehut tracker.", ) parser.add_argument( - '--srht-tracker', + "--srht-tracker", required=True, - help='Name of Sourcehut tracker to submit to.', + help="Name of Sourcehut tracker to submit to.", ) parser.add_argument( - '--gitlab-project-url', + "--gitlab-project-url", required=True, help="The base URL the project on Gitlab.", ) parser.add_argument( - '--mode', - default='print', + "--mode", + default="print", help="Action to take, 'print' or 'send'.", ) parser.add_argument( - '--from', + "--from", help="From address if mode is 'send'.", ) parser.add_argument( - '--smtp-host', + "--smtp-host", help="SMTP host to use.", ) parser.add_argument( - '--smtp-port', + "--smtp-port", default=None, help="SMTP port to use.", ) parser.add_argument( - '--smtp-ssl', - action='store_true', + "--smtp-ssl", + action="store_true", help="Use SMTP over SSL.", ) parser.add_argument( - '--smtp-starttls', - action='store_true', + "--smtp-starttls", + action="store_true", help="Use STARTTLS.", ) parser.add_argument( - '--smtp-user', + "--smtp-user", help="SMTP username.", ) parser.add_argument( - '--smtp-password', + "--smtp-password", help="SMTP password.", ) parser.add_argument( - '--smtp-delay', + "--smtp-delay", default=5, help="Decimal number of seconds to wait after sending each email.", ) parser.add_argument( - '--labels-file', + "--labels-file", help="CSV file mapping label IDs to names.", ) parser.add_argument( - '--skip-labels', - action='store_true', + "--skip-labels", + action="store_true", help="Skip mapping label IDs to names.", ) parser.add_argument( - '--skip-unknown-labels', - action='store_true', + "--skip-unknown-labels", + action="store_true", help="Skip mapping labels that aren't in the labels file.", ) parser.add_argument( - '--users-file', + "--users-file", help="CSV file mapping user IDs to names.", ) parser.add_argument( - '--skip-users', - action='store_true', + "--skip-users", + action="store_true", help="Skip mapping user IDs to names.", ) parser.add_argument( - '--skip-unknown-users', - action='store_true', + "--skip-unknown-users", + action="store_true", help="Skip mapping users that aren't in the users file.", ) parser.add_argument( - '--skip-missing-issues', - action='store_true', + "--skip-missing-issues", + action="store_true", help="Skip missing Gitlab issue IDs; GL and sr.ht IDs will not match.", ) parser.add_argument( - '--create-missing-issues', - action='store_true', + "--create-missing-issues", + action="store_true", help="Create missing GL issues in sr.ht to make issue IDs match.", ) parser.add_argument( - '--include-confidential', - action='store_true', + "--include-confidential", + action="store_true", help="Include confidential tickets and notes.", ) parser.add_argument( - '--skip-confidential', - action='store_true', + "--skip-confidential", + action="store_true", help="Skip confidential tickets and notes.", ) parser.add_argument( - 'export_dir', - help='Exported Gitlab tree/project/ directory containing ndjson files.', + "export_dir", + help="Exported Gitlab tree/project/ directory containing ndjson files.", ) args = vars(parser.parse_args()) - export_dir = args['export_dir'] + export_dir = args["export_dir"] assert export_dir, f"Must have a exported project directory." export_dir_path = Path(export_dir) - assert export_dir_path.is_dir(), \ - f"Project directory is not a directory: {export_dir_path}" - - mode = args['mode'] - frm = args['from'] - - labels_file = args['labels_file'] - skip_labels = args['skip_labels'] - skip_unknown_labels = args['skip_unknown_labels'] - assert labels_file or skip_labels, \ - f"One of --labels-file or --skip-labels must be provided." - - users_file = args['users_file'] - skip_users = args['skip_users'] - skip_unknown_users = args['skip_unknown_users'] - assert skip_users or users_file, \ - f"One of --users-file or --skip-users must be provided." - - skip_missing_issues = args['skip_missing_issues'] - create_missing_issues = args['create_missing_issues'] - assert not (skip_missing_issues and create_missing_issues), \ - f"Can accept at most one of --skip-missing-issues and --create-missing-issues." - - include_confidential = args['include_confidential'] - skip_confidential = args['skip_confidential'] - assert not (include_confidential and skip_confidential), \ - f"Can accept at most one of --include-confidential and --skip-confidential." - - if mode == 'print': + assert ( + export_dir_path.is_dir() + ), f"Project directory is not a directory: {export_dir_path}" + + mode = args["mode"] + frm = args["from"] + + labels_file = args["labels_file"] + skip_labels = args["skip_labels"] + skip_unknown_labels = args["skip_unknown_labels"] + assert ( + labels_file or skip_labels + ), f"One of --labels-file or --skip-labels must be provided." + + users_file = args["users_file"] + skip_users = args["skip_users"] + skip_unknown_users = args["skip_unknown_users"] + assert ( + skip_users or users_file + ), f"One of --users-file or --skip-users must be provided." + + skip_missing_issues = args["skip_missing_issues"] + create_missing_issues = args["create_missing_issues"] + assert not ( + skip_missing_issues and create_missing_issues + ), f"Can accept at most one of --skip-missing-issues and --create-missing-issues." + + include_confidential = args["include_confidential"] + skip_confidential = args["skip_confidential"] + assert not ( + include_confidential and skip_confidential + ), f"Can accept at most one of --include-confidential and --skip-confidential." + + if mode == "print": smtp = None - elif mode == 'send': - smtp_ssl = args['smtp_ssl'] - smtp_starttls = args['smtp_starttls'] - smtp_host = args['smtp_host'] or os.environ.get('SMTP_HOST', 'localhost') - smtp_port = args['smtp_port'] or os.environ.get('SMTP_PORT', 465 if smtp_ssl else 25) - smtp_user = args['smtp_user'] or os.environ.get('SMTP_USER', None) - smtp_password = args['smtp_password'] or os.environ.get('SMTP_PASSWORD', None) + elif mode == "send": + smtp_ssl = args["smtp_ssl"] + smtp_starttls = args["smtp_starttls"] + smtp_host = args["smtp_host"] or os.environ.get("SMTP_HOST", "localhost") + smtp_port = args["smtp_port"] or os.environ.get( + "SMTP_PORT", 465 if smtp_ssl else 25 + ) + smtp_user = args["smtp_user"] or os.environ.get("SMTP_USER", None) + smtp_password = args["smtp_password"] or os.environ.get("SMTP_PASSWORD", None) assert smtp_user, f"No SMTP user given." assert smtp_password, f"No SMTP password given." @@ -829,13 +848,13 @@ def main(): run( smtp=smtp, - smtp_delay=float(args['smtp_delay']), + smtp_delay=float(args["smtp_delay"]), mode=mode, - srht_owner=args['srht_owner'], - srht_tracker=args['srht_tracker'], + srht_owner=args["srht_owner"], + srht_tracker=args["srht_tracker"], frm=frm, export_dir_path=export_dir_path, - gitlab_project_url=args['gitlab_project_url'].rstrip('/'), + gitlab_project_url=args["gitlab_project_url"].rstrip("/"), labels_file_path=None if skip_labels else Path(labels_file), skip_unknown_labels=skip_unknown_labels, users_file_path=None if skip_users else Path(users_file), @@ -846,9 +865,9 @@ def main(): skip_confidential=skip_confidential, ) - if mode == 'send': + if mode == "send": smtp.quit() -if __name__ == '__main__': +if __name__ == "__main__": main() |