# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, # You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, unicode_literals import os import re import subprocess import sys import logging from mach.decorators import ( CommandArgument, CommandProvider, Command, ) from mozbuild.base import MachCommandBase import mozpack.path as mozpath import json GITHUB_ROOT = 'https://github.com/' PR_REPOSITORIES = { 'webrender': { 'github': 'servo/webrender', 'path': 'gfx/wr', 'bugzilla_product': 'Core', 'bugzilla_component': 'Graphics: WebRender', }, 'webgpu': { 'github': 'gfx-rs/wgpu', 'path': 'gfx/wgpu', 'bugzilla_product': 'Core', 'bugzilla_component': 'Graphics: WebGPU', }, 'debugger': { 'github': 'firefox-devtools/debugger', 'path': 'devtools/client/debugger', 'bugzilla_product': 'DevTools', 'bugzilla_component': 'Debugger' }, } @CommandProvider class PullRequestImporter(MachCommandBase): @Command('import-pr', category='misc', description='Import a pull request from Github to the local repo.') @CommandArgument('-b', '--bug-number', help='Bug number to use in the commit messages.') @CommandArgument('-t', '--bugzilla-token', help='Bugzilla API token used to file a new bug if no bug number is ' 'provided.') @CommandArgument('-r', '--reviewer', help='Reviewer nick to apply to commit messages.') @CommandArgument('pull_request', help='URL to the pull request to import (e.g. ' 'https://github.com/servo/webrender/pull/3665).') def import_pr(self, pull_request, bug_number=None, bugzilla_token=None, reviewer=None): import requests pr_number = None repository = None for r in PR_REPOSITORIES.values(): if pull_request.startswith(GITHUB_ROOT + r['github'] + '/pull/'): # sanitize URL, dropping anything after the PR number pr_number = int(re.search('/pull/([0-9]+)', pull_request).group(1)) pull_request = GITHUB_ROOT + r['github'] + '/pull/' + str(pr_number) repository = r break if repository is None: self.log(logging.ERROR, 'unrecognized_repo', {}, 'The pull request URL was not recognized; add it to the list of ' 'recognized repos in PR_REPOSITORIES in %s' % __file__) sys.exit(1) self.log(logging.INFO, 'import_pr', {'pr_url': pull_request}, 'Attempting to import {pr_url}') dirty = [f for f in self.repository.get_changed_files(mode='all') if f.startswith(repository['path'])] if dirty: self.log(logging.ERROR, 'dirty_tree', repository, 'Local {path} tree is dirty; aborting!') sys.exit(1) target_dir = mozpath.join(self.topsrcdir, os.path.normpath(repository['path'])) if bug_number is None: if bugzilla_token is None: self.log(logging.WARNING, 'no_token', {}, 'No bug number or bugzilla API token provided; bug number will not ' 'be added to commit messages.') else: bug_number = self._file_bug(bugzilla_token, repository, pr_number) elif bugzilla_token is not None: self.log(logging.WARNING, 'too_much_bug', {}, 'Providing a bugzilla token is unnecessary when a bug number is provided. ' 'Using bug number; ignoring token.') pr_patch = requests.get(pull_request + '.patch') pr_patch.raise_for_status() for patch in self._split_patches(pr_patch.content, bug_number, pull_request, reviewer): self.log(logging.INFO, 'commit_msg', patch, 'Processing commit [{commit_summary}] by [{author}] at [{date}]') patch_cmd = subprocess.Popen(['patch', '-p1', '-s'], stdin=subprocess.PIPE, cwd=target_dir) patch_cmd.stdin.write(patch['diff'].encode('utf-8')) patch_cmd.stdin.close() patch_cmd.wait() if patch_cmd.returncode != 0: self.log(logging.ERROR, 'commit_fail', {}, 'Error applying diff from commit via "patch -p1 -s". Aborting...') sys.exit(patch_cmd.returncode) self.repository.commit(patch['commit_msg'], patch['author'], patch['date'], [target_dir]) self.log(logging.INFO, 'commit_pass', {}, 'Committed successfully.') def _file_bug(self, token, repo, pr_number): import requests bug = requests.post('https://bugzilla.mozilla.org/rest/bug?api_key=%s' % token, json={ 'product': repo['bugzilla_product'], 'component': repo['bugzilla_component'], 'summary': 'Land %s#%s in mozilla-central' % (repo['github'], pr_number), 'version': 'unspecified', }) bug.raise_for_status() self.log(logging.DEBUG, 'new_bug', {}, bug.content) bugnumber = json.loads(bug.content)['id'] self.log(logging.INFO, 'new_bug', {'bugnumber': bugnumber}, 'Filed bug {bugnumber}') return bugnumber def _split_patches(self, patchfile, bug_number, pull_request, reviewer): INITIAL = 0 HEADERS = 1 STAT_AND_DIFF = 2 patch = b'' state = INITIAL for line in patchfile.splitlines(): if state == INITIAL: if line.startswith(b'From '): state = HEADERS elif state == HEADERS: patch += line + b'\n' if line == b'---': state = STAT_AND_DIFF elif state == STAT_AND_DIFF: if line.startswith(b'From '): yield self._parse_patch(patch, bug_number, pull_request, reviewer) patch = b'' state = HEADERS else: patch += line + b'\n' if len(patch) > 0: yield self._parse_patch(patch, bug_number, pull_request, reviewer) return def _parse_patch(self, patch, bug_number, pull_request, reviewer): import email from email import ( header, policy, ) parse_policy = policy.compat32.clone(max_line_length=None) parsed_mail = email.message_from_bytes(patch, policy=parse_policy) def header_as_unicode(key): decoded = header.decode_header(parsed_mail[key]) return str(header.make_header(decoded)) author = header_as_unicode('From') date = header_as_unicode('Date') commit_summary = header_as_unicode('Subject') email_body = parsed_mail.get_payload(decode=True).decode('utf-8') (commit_body, diff) = ('\n' + email_body).rsplit('\n---\n', 1) bug_prefix = '' if bug_number is not None: bug_prefix = 'Bug %s - ' % bug_number commit_summary = re.sub(r'^\[PATCH[0-9 /]*\] ', bug_prefix, commit_summary) if reviewer is not None: commit_summary += ' r=' + reviewer commit_msg = commit_summary + '\n' if len(commit_body) > 0: commit_msg += commit_body + '\n' commit_msg += '\n[import_pr] From ' + pull_request + '\n' patch_obj = { 'author': author, 'date': date, 'commit_summary': commit_summary, 'commit_msg': commit_msg, 'diff': diff, } return patch_obj