3
move_mrs_to_monorepo.py
source link: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/blob/main/scripts/move_mrs_to_monorepo.py
Go to the source link to view the article. You can view the picture content, updated content and better typesetting reading experience. If the link is broken, please click the button below to view the snapshot at that time.
scripts/move_mrs_to_monorepo.py · main · GStreamer
move_mrs_to_monorepo.py
22.4 KB
#!/usr/bin/env python3
from urllib.parse import urlparse
from contextlib import contextmanager
import os
import re
import sys
try:
import gitlab
except ModuleNotFoundError:
print("========================================================================", file=sys.stderr)
print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab dateutil`", file=sys.stderr)
print("========================================================================", file=sys.stderr)
sys.exit(1)
try:
from dateutil import parser as dateparse
except ModuleNotFoundError:
print("========================================================================", file=sys.stderr)
print("ERROR: Install dateutil with `python3 -m pip install dateutil`", file=sys.stderr)
print("========================================================================", file=sys.stderr)
sys.exit(1)
import argparse
import requests
import subprocess
ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
URL = "https://gitlab.freedesktop.org/"
SIGN_IN_URL = URL + 'sign_in'
LOGIN_URL = URL + 'users/sign_in'
LOGIN_URL_LDAP = URL + '/users/auth/ldapmain/callback'
MONOREPO_REMOTE_NAME = 'origin'
NAMESPACE = "gstreamer"
MONOREPO_NAME = 'gstreamer'
MONOREPO_REMOTE = URL + f'{NAMESPACE}/{MONOREPO_NAME}'
MONOREPO_BRANCH = 'main'
PING_SIGN = '@'
MOVING_NAMESPACE = NAMESPACE
PARSER = argparse.ArgumentParser(
description="Move merge request from old GStreamer module to the new"
"GStreamer 'monorepo'.\n"
" All your pending merge requests from all GStreamer modules will"
" be moved the the mono repository."
)
PARSER.add_argument("--skip-branch", action="store", nargs="*",
help="Ignore MRs for branches which match those names.", dest="skipped_branches")
PARSER.add_argument("--skip-on-failure", action="store_true", default=False)
PARSER.add_argument("--dry-run", "-n", action="store_true", default=False)
PARSER.add_argument("--use-branch-if-exists", action="store_true", default=False)
PARSER.add_argument(
"-c",
"--config-file",
action="append",
dest='config_files',
help="Configuration file to use. Can be used multiple times.",
required=False,
)
PARSER.add_argument(
"-g",
"--gitlab",
help=(
"Which configuration section should "
"be used. If not defined, the default selection "
"will be used."
),
required=False,
)
PARSER.add_argument(
"-m",
"--module",
help="GStreamer module to move MRs for. All if none specified. Can be used multiple times.",
dest='modules',
action="append",
required=False,
)
PARSER.add_argument(
"--mr",
default=None,
type=int,
help=(
"Id of the MR to work on."
" One (and only one) module must be specified with `--module`."
),
required=False,
)
GST_PROJECTS = [
'gstreamer',
'gst-plugins-base',
'gst-plugins-good',
'gst-plugins-bad',
'gst-plugins-ugly',
'gst-libav',
'gst-rtsp-server',
'gstreamer-vaapi',
'gstreamer-sharp',
'gst-python',
'gst-omx',
'gst-editing-services',
'gst-devtools',
'gst-integration-testsuites',
'gst-docs',
'gst-examples',
'gst-build',
'gst-ci',
]
# We do not want to deal with LFS
os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"
log_depth = [] # type: T.List[str]
@contextmanager
def nested(name=''):
global log_depth
log_depth.append(name)
try:
yield
finally:
log_depth.pop()
def bold(text: str):
return f"\033[1m{text}\033[0m"
def green(text: str):
return f"\033[1;32m{text}\033[0m"
def red(text: str):
return f"\033[1;31m{text}\033[0m"
def yellow(text: str):
return f"\033[1;33m{text}\033[0m"
def fprint(msg, nested=True):
if log_depth:
prepend = log_depth[-1] + ' | ' if nested else ''
else:
prepend = ''
print(prepend + msg, end="")
sys.stdout.flush()
class GstMRMover:
def __init__(self):
self.modules = []
self.gitlab = None
self.config_files = []
self.gl = None
self.mr = None
self.all_projects = []
self.skipped_branches = []
self.git_rename_limit = None
self.skip_on_failure = None
self.dry_run = False
def connect(self):
fprint("Logging into gitlab...")
if self.gitlab:
gl = gitlab.Gitlab.from_config(self.gitlab, self.config_files)
fprint(f"{green(' OK')}\n", nested=False)
return gl
gitlab_api_token = os.environ.get('GITLAB_API_TOKEN')
if gitlab_api_token:
gl = gitlab.Gitlab(URL, private_token=gitlab_api_token)
fprint(f"{green(' OK')}\n", nested=False)
return gl
session = requests.Session()
sign_in_page = session.get(SIGN_IN_URL).content.decode()
for l in sign_in_page.split('\n'):
m = re.search('name="authenticity_token" value="([^"]+)"', l)
if m:
break
token = None
if m:
token = m.group(1)
if not token:
fprint(f"{red('Unable to find the authenticity token')}\n")
sys.exit(1)
for data, url in [
({'user[login]': 'login_or_email',
'user[password]': 'SECRET',
'authenticity_token': token}, LOGIN_URL),
({'username': 'login_or_email',
'password': 'SECRET',
'authenticity_token': token}, LOGIN_URL_LDAP)]:
r = session.post(url, data=data)
if r.status_code != 200:
continue
try:
gl = gitlab.Gitlab(URL, api_version=4, session=session)
gl.auth()
except gitlab.exceptions.GitlabAuthenticationError as e:
continue
return gl
sys.exit(bold(f"{red('FAILED')}.\n\nPlease go to:\n\n"
' https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n'
f'and generate a token {bold("with read/write access to all but the registry")},'
' then set it in the "GITLAB_API_TOKEN" environment variable:"'
f'\n\n $ GITLAB_API_TOKEN=<your token> {" ".join(sys.argv)}\n'))
def git(self, *args, can_fail=False, interaction_message=None, call=False, revert_operation=None):
cwd = ROOT_DIR
retry = True
while retry:
retry = False
try:
if not call:
try:
return subprocess.check_output(["git"] + list(args), cwd=cwd,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT).decode()
except:
if not can_fail:
fprint(f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False)
raise
else:
subprocess.call(["git"] + list(args), cwd=cwd)
return "All good"
except Exception as e:
if interaction_message:
if self.skip_on_failure:
return "SKIP"
output = getattr(e, "output", b"")
if output is not None:
out = output.decode()
else:
out = "????"
fprint(f"\n```"
f"\n{out}\n"
f"Entering a shell in {cwd} to fix:\n\n"
f" {bold(interaction_message)}\n\n"
f"You should then exit with the following codes:\n\n"
f" - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n"
f" - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n"
f" - {bold('`exit 2`')}: to skip that merge request\n"
f" - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n"
"\n```\n", nested=False)
try:
if os.name == 'nt':
shell = os.environ.get(
"COMSPEC", r"C:\WINDOWS\system32\cmd.exe")
else:
shell = os.environ.get(
"SHELL", os.path.realpath("/bin/sh"))
subprocess.check_call(shell, cwd=cwd)
except subprocess.CalledProcessError as e:
if e.returncode == 1:
retry = True
continue
elif e.returncode == 2:
if revert_operation:
self.git(*revert_operation, can_fail=True)
return "SKIP"
elif e.returncode == 3:
if revert_operation:
self.git(*revert_operation, can_fail=True)
sys.exit(3)
except:
# Result of subshell does not really matter
pass
return "User fixed it"
if can_fail:
return "Failed but we do not care"
raise e
def cleanup_args(self):
if not self.modules:
if self.mr:
sys.exit(f"{red(f'Merge request #{self.mr} specified without module')}\n\n"
f"{bold(' -> Use `--module` to specify which module the MR is from.')}")
self.modules = GST_PROJECTS
else:
VALID_PROJECTS = GST_PROJECTS[1:]
for m in self.modules:
if m not in VALID_PROJECTS:
projects = '\n- '.join(VALID_PROJECTS)
sys.exit(f"{red(f'Unknown module {m}')}\nModules are:\n- {projects}")
if self.mr and len(self.modules) > 1:
sys.exit(f"{red(f'Merge request #{self.mr} specified but several modules where specified')}\n\n"
f"{bold(' -> Use `--module` only once to specify an merge request.')}")
self.modules.append(GST_PROJECTS[0])
def run(self):
self.cleanup_args()
self.gl = self.connect()
self.gl.auth()
try:
prevbranch = self.git("rev-parse", "--abbrev-ref", "HEAD", can_fail=True).strip()
except:
fprint(bold(yellow("Not on a branch?\n")), indent=False)
prevbranch = None
try:
self.setup_repo()
from_projects, to_project = self.fetch_projects()
with nested(' '):
self.move_mrs(from_projects, to_project)
finally:
if self.git_rename_limit is not None:
self.git("config", "merge.renameLimit", str(self.git_rename_limit))
if prevbranch:
fprint(f'Back to {prevbranch}\n')
self.git("checkout", prevbranch)
def fetch_projects(self):
fprint("Fetching projects... ")
self.all_projects = [proj for proj in self.gl.projects.list(
membership=1, all=True) if proj.name in self.modules]
self.user_project, = [p for p in self.all_projects
if p.namespace['path'] == self.gl.user.username
and p.name == MONOREPO_NAME]
fprint(f"{green(' OK')}\n", nested=False)
from_projects = [proj for proj in self.all_projects if proj.namespace['path']
== NAMESPACE and proj.name != "gstreamer"]
fprint(f"\nMoving MRs from:\n")
fprint(f"----------------\n")
for p in from_projects:
fprint(f" - {bold(p.path_with_namespace)}\n")
to_project, = [p for p in self.all_projects if p.path_with_namespace ==
MOVING_NAMESPACE + "/gstreamer"]
fprint(f"To: {bold(to_project.path_with_namespace)}\n\n")
return from_projects, to_project
def recreate_mr(self, project, to_project, mr):
branch = f"{project.name}-{mr.source_branch}"
if not self.create_branch_for_mr(branch, project, mr):
return None
description = f"**Copied from {URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}**\n\n{mr.description}"
title = mr.title
if ':' not in mr.title:
title = f"{project.name}: {mr.title}"
new_mr_dict = {
'source_branch': branch,
'allow_collaboration': True,
'remove_source_branch': True,
'target_project_id': to_project.id,
'target_branch': MONOREPO_BRANCH,
'title': title,
'labels': mr.labels,
'description': description,
}
try:
fprint(f"-> Recreating MR '{bold(mr.title)}'...")
if self.dry_run:
fprint(f"\nDry info:\n{new_mr_dict}\n")
else:
new_mr = self.user_project.mergerequests.create(new_mr_dict)
fprint(f"{green(' OK')}\n", nested=False)
except gitlab.exceptions.GitlabCreateError as e:
fprint(f"{yellow('SKIPPED')} (An MR already exists)\n", nested=False)
return None
fprint(f"-> Adding discussings from MR '{mr.title}'...")
if self.dry_run:
fprint(f"{green(' OK')}\n", nested=False)
return None
new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
for issue in mr.closes_issues():
obj = {'body': f'Fixing MR moved to: {new_mr_url}'}
issue.discussions.create(obj)
mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
for discussion in mr.discussions.list():
# FIXME notes = [n for n in discussion.attributes['notes'] if n['type'] is not None]
notes = [n for n in discussion.attributes['notes']]
if not notes:
continue
new_discussion = None
for note in notes:
note = discussion.notes.get(note['id'])
note_url = f"{mr_url}#note_{note.id}"
when = dateparse.parse(note.created_at).strftime('on %d, %b %Y')
body = f"**{note.author['name']} - {PING_SIGN}{note.author['username']} wrote [here]({note_url})** {when}:\n\n"
body += '\n'.join([l for l in note.body.split('\n')])
obj = {
'body': body,
'type': note.type,
'resolvable': note.resolvable,
}
if new_discussion:
new_discussion.notes.create(obj)
else:
new_discussion = new_mr.discussions.create(obj)
if not note.resolvable or note.resolved:
new_discussion.resolved = True
new_discussion.save()
fprint(f"{green(' OK')}\n", nested=False)
print(f"New MR available at: {bold(new_mr_url)}\n")
return new_mr
def push_branch(self, branch):
fprint(f"-> Pushing branch {branch} to remote {self.gl.user.username}...")
if self.git("push", "--no-verify", self.gl.user.username, branch,
interaction_message=f"pushing {branch} to {self.gl.user.username} with:\n "
f" `$git push {self.gl.user.username} {branch}`") == "SKIP":
fprint(yellow("'SKIPPED' (couldn't push)"), nested=False)
return False
fprint(f"{green(' OK')}\n", nested=False)
return True
def create_branch_for_mr(self, branch, project, mr):
remote_name = project.name + '-' + self.gl.user.username
remote_branch = f"{MONOREPO_REMOTE_NAME}/{MONOREPO_BRANCH}"
if self.use_branch_if_exists:
try:
self.git("checkout", branch)
self.git("show", remote_branch + "..", call=True)
if self.dry_run:
fprint("Dry run... not creating MR")
return True
cont = input('\n Create MR [y/n]? ')
if cont.strip().lower() != 'y':
fprint("Cancelled")
return False
return self.push_branch(branch)
except subprocess.CalledProcessError as e:
pass
self.git("remote", "add", remote_name,
f"{URL}{self.gl.user.username}/{project.name}.git", can_fail=True)
self.git("fetch", remote_name)
if self.git("checkout", remote_branch, "-b", branch,
interaction_message=f"checking out branch with `git checkout {remote_branch} -b {branch}`") == "SKIP":
fprint(bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False)
return False
for commit in reversed([c for c in mr.commits()]):
if self.git("cherry-pick", commit.id,
interaction_message=f"cherry-picking {commit.id} onto {branch} with:\n "
f" `$ git cherry-pick {commit.id}`",
revert_operation=["cherry-pick", "--abort"]) == "SKIP":
fprint(f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False)
return False
self.git("show", remote_branch + "..", call=True)
if self.dry_run:
fprint("Dry run... not creating MR\n")
return True
cont = input('\n Create MR [y/n]? ')
if cont.strip().lower() != 'y':
fprint(f"{red('Cancelled')}\n", nested=False)
return False
return self.push_branch(branch)
def move_mrs(self, from_projects, to_project):
failed_mrs = []
found_mr = None
for from_project in from_projects:
with nested(f'{bold(from_project.path_with_namespace)}'):
fprint(f'Fetching mrs')
mrs = [mr for mr in from_project.mergerequests.list(
all=True, author_id=self.gl.user.id) if mr.author['username'] == self.gl.user.username and mr.state == "opened"]
if not mrs:
fprint(f"{yellow(' None')}\n", nested=False)
continue
fprint(f"{green(' DONE')}\n", nested=False)
for mr in mrs:
if self.mr:
if self.mr != mr.iid:
continue
found_mr = True
fprint(f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ')
if mr.source_branch in self.skipped_branches:
print(f"{yellow('SKIPPED')} (blacklisted branch)")
failed_mrs.append(
f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
continue
with nested(f'{bold(from_project.path_with_namespace)}: {mr.iid}'):
new_mr = self.recreate_mr(from_project, to_project, mr)
if not new_mr:
if not self.dry_run:
failed_mrs.append(
f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
else:
fprint(f"{green(' OK')}\n", nested=False)
self.close_mr(from_project, to_project, mr, new_mr)
fprint(f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False)
if self.mr and not found_mr:
sys.exit(bold(red(f"\n==> Couldn't find MR {self.mr} in {self.modules[0]}\n")))
for mr in failed_mrs:
fprint(f"Didn't move MR: {mr}\n")
def close_mr(self, project, to_project, mr, new_mr):
if new_mr:
new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
else:
new_mr_url = None
mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
cont = input(f'\n Close old MR {mr_url} "{bold(mr.title)}" ? [y/n]')
if cont.strip().lower() != 'y':
fprint(f"{yellow('Not closing old MR')}\n")
else:
obj = None
if new_mr_url:
obj = {'body': f"Moved to: {new_mr_url}"}
else:
ret = input(f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip()
if ret:
obj = {'body': ret}
if self.dry_run:
fprint(f"{bold('Dry run, not closing')}\n", nested=False)
else:
if obj:
mr.discussions.create(obj)
mr.state_event = 'close'
mr.save()
fprint(f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n')
def setup_repo(self):
fprint(f"Setting up '{bold(ROOT_DIR)}'...")
try:
out = self.git("status", "--porcelain")
if out:
fprint("\n" + red('Git repository is not clean:') + "\n```\n" + out + "\n```\n")
sys.exit(1)
except Exception as e:
exit(
f"Git repository{ROOT_DIR} is not clean. Clean it up before running {sys.argv[0]}\n ({e})")
self.git('remote', 'add', MONOREPO_REMOTE_NAME,
MONOREPO_REMOTE, can_fail=True)
self.git('fetch', MONOREPO_REMOTE_NAME)
self.git('remote', 'add', self.gl.user.username,
f"[email protected]:{self.gl.user.username}/gstreamer.git", can_fail=True)
self.git('fetch', self.gl.user.username,
interaction_message=f"Setup your fork of {URL}gstreamer/gstreamer as remote called {self.gl.user.username}")
fprint(f"{green(' OK')}\n", nested=False)
try:
git_rename_limit = int(self.git("config", "merge.renameLimit"))
except subprocess.CalledProcessError:
git_rename_limit = 0
if int(git_rename_limit) < 999999:
self.git_rename_limit = git_rename_limit
fprint("-> Setting git rename limit to 999999 so we can properly cherry-pick between repos\n")
self.git("config", "merge.renameLimit", "999999")
def main():
mover = GstMRMover()
PARSER.parse_args(namespace=mover)
mover.run()
if __name__ == '__main__':
main()
Recommend
About Joyk
Aggregate valuable and interesting links.
Joyk means Joy of geeK