#! /usr/bin/env python3 assert __name__ == '__main__' ''' To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come before python3 in your path. Upstream: https://chromium.googlesource.com/angle/angle Our repo: https://github.com/mozilla/angle It has branches like 'firefox-60' which is the branch we use for pulling into Gecko with this script. This script leaves a record of the merge-base and cherry-picks that we pull into Gecko. (gfx/angle/cherries.log) ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/ An easy choice is to grab Chrome's Beta's ANGLE branch. ## Usage Prepare your env: ~~~ export PATH="$PATH:/path/to/depot_tools" export DEPOT_TOOLS_WIN_TOOLCHAIN=0 ~~~ If this is a new repo, don't forget: ~~~ # In the angle repo: ./scripts/bootstrap.py gclient sync ~~~ Update: (in the angle repo) ~~~ # In the angle repo: /path/to/gecko/gfx/angle/update-angle.py origin/chromium/XXXX git push moz # Push the firefox-XX branch to github.com/mozilla/angle ~~~~ ''' import json import os from pathlib import * import re import shutil import subprocess import sys from vendor_from_git import * REPO_DIR = Path.cwd() GECKO_ANGLE_DIR = Path(__file__).parent OUT_DIR = 'out' COMMON_HEADER = [ '# Generated by update-angle.py', '', "include('../../moz.build.common')", ] VENDOR_PREREQ_TARGETS = [ '//:commit_id', # Generate 'commit.h'. ] ROOTS = ['//:translator', '//:libEGL', '//:libGLESv2'] DRY_RUN = '--dry' in sys.argv ACTION_PREFIX = '' if DRY_RUN: ACTION_PREFIX = '(not) ' # -------------------------------------- def sorted_items(x): for k in sorted(x.keys()): yield (k, x[k]) def collapse_dotdots(path): split = path.split('/') ret = [] for x in split: if x == '..' and ret: ret.pop() continue ret.append(x) continue return '/'.join(ret) def traverse(roots, pre_recurse_func, key_func=id): visited = set() def recurse(cur): key = key_func(cur) if key in visited: return visited.add(key) t = pre_recurse_func(cur) post_recurse_func = None try: (children, post_recurse_func) = t except ValueError: (children,) = t for x in children: recurse(x) if post_recurse_func: post_recurse_func(cur) return for x in roots: recurse(x) return # -------------------------------------- MERGE_BASE = sys.argv[1] if not DRY_RUN: record_cherry_picks(GECKO_ANGLE_DIR, MERGE_BASE) # -- print_now('Importing graph') shutil.rmtree(OUT_DIR, True) run_checked('gn', 'gen', OUT_DIR, shell=True) GN_ARGS = ''' # Build arguments go here. # See "gn args --list" for available build arguments. is_clang = false angle_enable_gl = false angle_enable_gl_null = false angle_enable_null = false angle_enable_vulkan = false '''[1:] with open(OUT_DIR + '/args.gn', 'wb') as f: f.write(GN_ARGS.encode()) # -- p = run_checked('gn', 'desc', '--format=json', OUT_DIR, '*', stdout=subprocess.PIPE, shell=True) print_now('Processing graph') descs = json.loads(p.stdout.decode()) # HACKHACKHACK common = descs['//:angle_common'] common['sources'] += [ '//src/common/system_utils_linux.cpp', '//src/common/system_utils_mac.cpp', ] # -- for (k, v) in descs.items(): for (k2, v2) in v.items(): if type(v2) == list: v[k2] = tuple(v2) # Freeze lists v['target_name'] = k v['dep_nodes'] = tuple([descs[x] for x in v['deps']]) assert v['public'] == '*', k # -- # Ready to traverse ROOTS = [descs[k] for k in ROOTS] # Gather real targets: real_targets = [] def gather_real_targets(cur): print_now(' ' + cur['type'], cur['target_name']) if cur['type'] in ['shared_library', 'static_library']: real_targets.append(cur) def post(x): x['sources_with_deps'] = x.get('sources', ()) x['include_dirs_with_deps'] = x.get('include_dirs', ()) for y in x['dep_nodes']: x['sources_with_deps'] += y['sources_with_deps'] x['include_dirs_with_deps'] += y['include_dirs_with_deps'] return (cur['dep_nodes'], post) traverse(ROOTS, gather_real_targets) # -- def sortedi(x): return sorted(x, key=str.lower) def append_arr(dest, name, vals, indent=0): if not vals: return dest.append('{}{} += ['.format(' '*4*indent, name)) for x in sortedi(vals): dest.append("{}'{}',".format(' '*4*(indent+1), x)) dest.append('{}]'.format(' '*4*indent)) dest.append('') return INCLUDE_REGEX = re.compile('# *include +([<"])([^>"]+)[>"]') IGNORED_INCLUDES = { 'compiler/translator/TranslatorVulkan.h', 'libANGLE/renderer/d3d/d3d11/winrt/NativeWindow11WinRT.h', 'libANGLE/renderer/gl/glx/DisplayGLX.h', 'libANGLE/renderer/gl/cgl/DisplayCGL.h', 'libANGLE/renderer/gl/egl/ozone/DisplayOzone.h', 'libANGLE/renderer/gl/egl/android/DisplayAndroid.h', 'libANGLE/renderer/gl/wgl/DisplayWGL.h', 'libANGLE/renderer/null/DisplayNULL.h', 'libANGLE/renderer/vulkan/android/DisplayVkAndroid.h', 'libANGLE/renderer/vulkan/win32/DisplayVkWin32.h', 'libANGLE/renderer/vulkan/xcb/DisplayVkXcb.h', } IGNORED_INCLUDE_PREFIXES = { 'android/', 'Carbon/', 'CoreFoundation/', 'CoreServices/', 'IOSurface/', 'mach/', 'mach-o/', 'OpenGL/', 'pci/', 'sys/', 'wrl/', 'X11/', } REGISTERED_DEFINES = { 'ANGLE_ENABLE_D3D11': True, 'ANGLE_ENABLE_D3D9': True, 'ANGLE_ENABLE_DEBUG_ANNOTATIONS': True, 'ANGLE_ENABLE_NULL': False, 'ANGLE_ENABLE_OPENGL': False, 'ANGLE_ENABLE_OPENGL_NULL': False, 'ANGLE_ENABLE_ESSL': True, 'ANGLE_ENABLE_GLSL': True, 'ANGLE_ENABLE_HLSL': True, 'ANGLE_GENERATE_SHADER_DEBUG_INFO': True, 'ANGLE_IS_64_BIT_CPU': False, 'ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES': False, 'CERT_CHAIN_PARA_HAS_EXTRA_FIELDS': False, 'CHROMIUM_BUILD': False, 'COMPONENT_BUILD': False, 'constexpr14': False, 'DYNAMIC_ANNOTATIONS_ENABLED': True, 'EGL_EGLEXT_PROTOTYPES': True, 'EGLAPI': True, 'FIELDTRIAL_TESTING_ENABLED': False, 'FULL_SAFE_BROWSING': False, 'GL_API': True, 'GL_APICALL': True, 'GL_GLEXT_PROTOTYPES': True, 'GPU_INFO_USE_SETUPAPI': True, 'LIBANGLE_IMPLEMENTATION': True, 'LIBEGL_IMPLEMENTATION': True, 'LIBGLESV2_IMPLEMENTATION': True, 'NOMINMAX': True, 'NO_TCMALLOC': False, # Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object 'NTDDI_VERSION': True, 'PSAPI_VERSION': False, 'SAFE_BROWSING_CSD': False, 'SAFE_BROWSING_DB_LOCAL': False, 'UNICODE': True, 'USE_AURA': False, 'V8_DEPRECATION_WARNINGS': False, 'WIN32': False, 'WIN32_LEAN_AND_MEAN': False, 'WINAPI_FAMILY': False, 'WINVER': False, 'WTF_USE_DYNAMIC_ANNOTATIONS': False, '_ATL_NO_OPENGL': True, '_CRT_RAND_S': True, '_CRT_SECURE_NO_DEPRECATE': True, '_DEBUG': False, '_HAS_EXCEPTIONS': True, '_SCL_SECURE_NO_DEPRECATE': True, '_SECURE_ATL': True, '_UNICODE': True, '_USING_V110_SDK71_': False, '_WIN32_WINNT': False, '_WINDOWS': False, '__STD_C': False, } SOURCE_FILE_EXTS = frozenset(['h', 'hpp', 'inc', 'inl', 'c', 'cc', 'cpp']) def is_source_file(x): e = x.split('.')[-1] return e in SOURCE_FILE_EXTS def assert_valid_includes(target_name, cur, avail_files, include_dirs): assert cur.startswith('//'), cur cur = PurePosixPath(cur[2:]) (cur_dir, _) = os.path.split(cur) include_dirs = [ '//', '//' + cur_dir + '/', ] + list(include_dirs) def assert_one(inc, line_num): attempts = [] for inc_dir in include_dirs: assert inc_dir[-1] == '/' inc_path = inc_dir + inc inc_path = collapse_dotdots(inc_path) attempts.append(inc_path) if inc_path in avail_files: return print('Warning in {}: {}:{}: Invalid include: {}'.format(target_name, cur, line_num, inc)) print(' Tried:') for x in attempts: print(' {}'.format(x)) #print() #print(avail_files) exit(1) line_num = 0 with open(cur, 'rb') as f: for line in f: line = line.decode() line_num += 1 m = INCLUDE_REGEX.match(line) if not m: continue inc = m.group(2) if inc in IGNORED_INCLUDES: continue if m.group(1) == '<': if '/' not in inc: continue if any((inc.startswith(x) for x in IGNORED_INCLUDE_PREFIXES)): continue assert_one(inc, line_num) total_used_files = set() vendor_prereq_outputs = set() # -- print_now('Running prerequisite actions') for k in VENDOR_PREREQ_TARGETS: assert k.startswith('//') run_checked('ninja', '-C', OUT_DIR, k[2:]) vendor_prereq_outputs |= set(descs[k]['outputs']) total_used_files |= vendor_prereq_outputs # -- # Export our targets print_now('Export targets') # Clear our dest directories targets_dir = Path(GECKO_ANGLE_DIR, 'targets') checkout_dir = Path(GECKO_ANGLE_DIR, 'checkout') if not DRY_RUN: shutil.rmtree(targets_dir, True) shutil.rmtree(checkout_dir, True) targets_dir.mkdir(exist_ok=True) checkout_dir.mkdir(exist_ok=True) def export_target(root): name = root['target_name'] assert name.startswith('//:') name = name[3:] used_files = root['sources_with_deps'] used_files = [x for x in used_files if x.split('.')[-1] not in ['dll']] global total_used_files total_used_files |= set(used_files) # Check includes, since `gn check` seems to be broken. includable = set(root['sources_with_deps']) | vendor_prereq_outputs for x in includable: if is_source_file(x): assert_valid_includes(name, x, includable, root['include_dirs_with_deps']) # Accumulate a combined dict for the target including non-lib deps. accum_desc = dict(root) del accum_desc['dep_nodes'] use_libs = set() checkable_sources = set() target_includable_files = set() def pre(cur): assert not cur.get('allow_circular_includes_from', ()), cur['target_name'] deps = cur['dep_nodes'] if cur != root: if cur['type'] in ['shared_library', 'static_library']: deps = [] name = cur['target_name'] assert name.startswith('//:') name = name[3:] use_libs.add(name) elif cur['type'] in ('source_set', 'group', 'action'): for (k,v) in cur.items(): if k in ('dep_nodes', 'sources_with_deps', 'include_dirs_with_deps'): continue if type(v) in (list, tuple): vs = accum_desc.setdefault(k, ()) vs += v else: accum_desc.setdefault(k, v) return (deps,) traverse([root], pre) # Create our manifest lines target_dir = Path(targets_dir, name) target_dir.mkdir(exist_ok=True) lines = COMMON_HEADER[:] lines.append('') for x in sorted(set(accum_desc['defines'])): try: (k, v) = x.split('=', 1) v = "'{}'".format(v) except ValueError: (k, v) = (x, 'True') try: line = "DEFINES['{}'] = {}".format(k, v) if REGISTERED_DEFINES[k] == False: line = '#' + line lines.append(line) except KeyError: print('[{}] Unrecognized define: {}'.format(name, k)) lines.append('') cxxflags = set(accum_desc['cflags'] + accum_desc['cflags_cc']) def fixup_paths(listt): for x in set(listt): assert x.startswith('//'), x yield '../../checkout/' + x[2:] sources_by_config = {} extras = dict() for x in fixup_paths(accum_desc['sources']): (b, e) = x.rsplit('.', 1) if e in ['h', 'y', 'l', 'inc', 'inl']: continue elif e in ['cpp', 'cc']: if b.endswith('_win'): config = "CONFIG['OS_ARCH'] == 'WINNT'" elif b.endswith('_linux'): # Include these on BSDs too. config = "CONFIG['OS_ARCH'] not in ('Darwin', 'WINNT')" elif b.endswith('_mac'): config = "CONFIG['OS_ARCH'] == 'Darwin'" else: config = '' # None can't compare against str. sources_by_config.setdefault(config, []).append(x) continue elif e == 'rc': assert 'RCFILE' not in extras extras['RCFILE'] = "'{}'".format(x) continue else: assert False, "Unhandled ext: {}".format(x) ldflags = set(accum_desc['ldflags']) DEF_PREFIX = '/DEF:' for x in set(ldflags): if x.startswith(DEF_PREFIX): assert 'DEFFILE' not in extras ldflags.remove(x) def_path = OUT_DIR + '/' + x[len(DEF_PREFIX):] def_path = '//' + collapse_dotdots(def_path) total_used_files.add(def_path) def_rel_path = list(fixup_paths([def_path]))[0] extras['DEFFILE'] = "SRCDIR + '/{}'".format(def_rel_path) os_libs = list(map( lambda x: x[:-len('.lib')], set(accum_desc.get('libs', [])) )) def append_arr_commented(dest, name, src): lines = [] append_arr(lines, name, src) def comment(x): if x: x = '#' + x return x lines = map(comment, lines) dest += lines append_arr(lines, 'LOCAL_INCLUDES', fixup_paths(accum_desc['include_dirs'])) append_arr_commented(lines, 'CXXFLAGS', cxxflags) for (config,v) in sorted_items(sources_by_config): indent = 0 if config: lines.append("if {}:".format(config)) indent = 1 append_arr(lines, 'SOURCES', v, indent=indent) append_arr(lines, 'USE_LIBS', use_libs) append_arr(lines, 'DIRS', ['../' + x for x in use_libs]) append_arr(lines, 'OS_LIBS', os_libs) append_arr_commented(lines, 'LDFLAGS', ldflags) for (k,v) in sorted(extras.items()): lines.append('{} = {}'.format(k, v)) lib_type = root['type'] if lib_type == 'shared_library': lines.append("GeckoSharedLibrary('{}', linkage=None)".format(name)) elif lib_type == 'static_library': lines.append("Library('{}')".format(name)) else: assert False, lib_type # Write it out mozbuild = Path(target_dir, 'moz.build') print_now(' {}Writing {}'.format(ACTION_PREFIX, mozbuild)) if not DRY_RUN: with mozbuild.open('w', newline='\n') as f: for x in lines: f.write(x + '\n') return for x in real_targets: export_target(x) # Copy all the files print_now('Migrate files') total_used_files = sorted(total_used_files) i = 0 for x in total_used_files: i += 1 sys.stdout.write('\r {}Copying {}/{}'.format(ACTION_PREFIX, i, len(total_used_files))) sys.stdout.flush() assert x.startswith('//'), x x = x[2:] src = Path(REPO_DIR, x) dest = Path(checkout_dir, x) if not DRY_RUN: dest.parent.mkdir(parents=True, exist_ok=True) data = src.read_bytes() data = data.replace(b'\r\n', b'\n') dest.write_bytes(data) print('\nDone')