forked from mirrors/gecko-dev
05822ea753
Leaving one fix for an explicit review.
Lint python/mozbuild/{mozbuild,mozpack}. r=#build
Differential Revision: https://phabricator.services.mozilla.com/D26641
--HG--
extra : moz-landing-system : lando
386 lines
14 KiB
Python
386 lines
14 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
|
|
from collections import Iterable, OrderedDict
|
|
from types import StringTypes, ModuleType
|
|
|
|
import mozpack.path as mozpath
|
|
|
|
from mozbuild.util import (
|
|
FileAvoidWrite,
|
|
memoized_property,
|
|
ReadOnlyDict,
|
|
)
|
|
from mozbuild.shellutil import quote as shell_quote
|
|
|
|
|
|
if sys.version_info.major == 2:
|
|
text_type = unicode
|
|
else:
|
|
text_type = str
|
|
|
|
|
|
class BuildConfig(object):
|
|
"""Represents the output of configure."""
|
|
|
|
_CODE_CACHE = {}
|
|
|
|
def __init__(self):
|
|
self.topsrcdir = None
|
|
self.topobjdir = None
|
|
self.defines = {}
|
|
self.non_global_defines = []
|
|
self.substs = {}
|
|
self.files = []
|
|
self.mozconfig = None
|
|
|
|
@classmethod
|
|
def from_config_status(cls, path):
|
|
"""Create an instance from a config.status file."""
|
|
code_cache = cls._CODE_CACHE
|
|
mtime = os.path.getmtime(path)
|
|
|
|
# cache the compiled code as it can be reused
|
|
# we cache it the first time, or if the file changed
|
|
if path not in code_cache or code_cache[path][0] != mtime:
|
|
# Add config.status manually to sys.modules so it gets picked up by
|
|
# iter_modules_in_path() for automatic dependencies.
|
|
mod = ModuleType('config.status')
|
|
mod.__file__ = path
|
|
sys.modules['config.status'] = mod
|
|
|
|
with open(path, 'rt') as fh:
|
|
source = fh.read()
|
|
code_cache[path] = (
|
|
mtime,
|
|
compile(source, path, 'exec', dont_inherit=1)
|
|
)
|
|
|
|
g = {
|
|
'__builtins__': __builtins__,
|
|
'__file__': path,
|
|
}
|
|
l = {}
|
|
exec(code_cache[path][1], g, l)
|
|
|
|
config = BuildConfig()
|
|
|
|
for name in l['__all__']:
|
|
setattr(config, name, l[name])
|
|
|
|
return config
|
|
|
|
|
|
class ConfigEnvironment(object):
|
|
"""Perform actions associated with a configured but bare objdir.
|
|
|
|
The purpose of this class is to preprocess files from the source directory
|
|
and output results in the object directory.
|
|
|
|
There are two types of files: config files and config headers,
|
|
each treated through a different member function.
|
|
|
|
Creating a ConfigEnvironment requires a few arguments:
|
|
- topsrcdir and topobjdir are, respectively, the top source and
|
|
the top object directory.
|
|
- defines is a dict filled from AC_DEFINE and AC_DEFINE_UNQUOTED in
|
|
autoconf.
|
|
- non_global_defines are a list of names appearing in defines above
|
|
that are not meant to be exported in ACDEFINES (see below)
|
|
- substs is a dict filled from AC_SUBST in autoconf.
|
|
|
|
ConfigEnvironment automatically defines one additional substs variable
|
|
from all the defines not appearing in non_global_defines:
|
|
- ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
|
|
preprocessor command lines. The order in which defines were given
|
|
when creating the ConfigEnvironment is preserved.
|
|
and two other additional subst variables from all the other substs:
|
|
- ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted
|
|
order, for use in autoconf.mk. It includes ACDEFINES
|
|
Only substs with a VALUE are included, such that the resulting file
|
|
doesn't change when new empty substs are added.
|
|
This results in less invalidation of build dependencies in the case
|
|
of autoconf.mk..
|
|
- ALLEMPTYSUBSTS contains the substs with an empty value, in the form
|
|
NAME =.
|
|
|
|
ConfigEnvironment expects a "top_srcdir" subst to be set with the top
|
|
source directory, in msys format on windows. It is used to derive a
|
|
"srcdir" subst when treating config files. It can either be an absolute
|
|
path or a path relative to the topobjdir.
|
|
"""
|
|
|
|
def __init__(self, topsrcdir, topobjdir, defines=None,
|
|
non_global_defines=None, substs=None, source=None, mozconfig=None):
|
|
|
|
if not source:
|
|
source = mozpath.join(topobjdir, 'config.status')
|
|
self.source = source
|
|
self.defines = ReadOnlyDict(defines or {})
|
|
self.non_global_defines = non_global_defines or []
|
|
self.substs = dict(substs or {})
|
|
self.topsrcdir = mozpath.abspath(topsrcdir)
|
|
self.topobjdir = mozpath.abspath(topobjdir)
|
|
self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None
|
|
self.lib_prefix = self.substs.get('LIB_PREFIX', '')
|
|
self.rust_lib_prefix = self.substs.get('RUST_LIB_PREFIX', '')
|
|
if 'LIB_SUFFIX' in self.substs:
|
|
self.lib_suffix = '.%s' % self.substs['LIB_SUFFIX']
|
|
if 'RUST_LIB_SUFFIX' in self.substs:
|
|
self.rust_lib_suffix = '.%s' % self.substs['RUST_LIB_SUFFIX']
|
|
self.dll_prefix = self.substs.get('DLL_PREFIX', '')
|
|
self.dll_suffix = self.substs.get('DLL_SUFFIX', '')
|
|
self.host_dll_prefix = self.substs.get('HOST_DLL_PREFIX', '')
|
|
self.host_dll_suffix = self.substs.get('HOST_DLL_SUFFIX', '')
|
|
if self.substs.get('IMPORT_LIB_SUFFIX'):
|
|
self.import_prefix = self.lib_prefix
|
|
self.import_suffix = '.%s' % self.substs['IMPORT_LIB_SUFFIX']
|
|
else:
|
|
self.import_prefix = self.dll_prefix
|
|
self.import_suffix = self.dll_suffix
|
|
self.bin_suffix = self.substs.get('BIN_SUFFIX', '')
|
|
|
|
global_defines = [name for name in self.defines
|
|
if name not in self.non_global_defines]
|
|
self.substs["ACDEFINES"] = ' '.join(
|
|
[
|
|
'-D%s=%s' % (name, shell_quote(self.defines[name]).replace('$', '$$'))
|
|
for name in sorted(global_defines)
|
|
]
|
|
)
|
|
|
|
def serialize(name, obj):
|
|
if isinstance(obj, StringTypes):
|
|
return obj
|
|
if isinstance(obj, Iterable):
|
|
return ' '.join(obj)
|
|
raise Exception('Unhandled type %s for %s', type(obj), str(name))
|
|
self.substs['ALLSUBSTS'] = '\n'.join(
|
|
sorted([
|
|
'%s = %s' % (
|
|
name,
|
|
serialize(name, self.substs[name])
|
|
)
|
|
for name in self.substs if self.substs[name]
|
|
])
|
|
)
|
|
self.substs['ALLEMPTYSUBSTS'] = '\n'.join(
|
|
sorted([
|
|
'%s =' % name
|
|
for name in self.substs if not self.substs[name]
|
|
])
|
|
)
|
|
|
|
self.substs = ReadOnlyDict(self.substs)
|
|
|
|
self.external_source_dir = None
|
|
external = self.substs.get('EXTERNAL_SOURCE_DIR', '')
|
|
if external:
|
|
external = mozpath.normpath(external)
|
|
if not os.path.isabs(external):
|
|
external = mozpath.join(self.topsrcdir, external)
|
|
self.external_source_dir = mozpath.normpath(external)
|
|
|
|
# Populate a Unicode version of substs. This is an optimization to make
|
|
# moz.build reading faster, since each sandbox needs a Unicode version
|
|
# of these variables and doing it over a thousand times is a hotspot
|
|
# during sandbox execution!
|
|
# Bug 844509 tracks moving everything to Unicode.
|
|
self.substs_unicode = {}
|
|
|
|
def decode(v):
|
|
if not isinstance(v, text_type):
|
|
try:
|
|
return v.decode('utf-8')
|
|
except UnicodeDecodeError:
|
|
return v.decode('utf-8', 'replace')
|
|
|
|
for k, v in self.substs.items():
|
|
if not isinstance(v, StringTypes):
|
|
if isinstance(v, Iterable):
|
|
type(v)(decode(i) for i in v)
|
|
elif not isinstance(v, text_type):
|
|
v = decode(v)
|
|
|
|
self.substs_unicode[k] = v
|
|
|
|
self.substs_unicode = ReadOnlyDict(self.substs_unicode)
|
|
|
|
@property
|
|
def is_artifact_build(self):
|
|
return self.substs.get('MOZ_ARTIFACT_BUILDS', False)
|
|
|
|
@memoized_property
|
|
def acdefines(self):
|
|
acdefines = dict((name, self.defines[name])
|
|
for name in self.defines
|
|
if name not in self.non_global_defines)
|
|
return ReadOnlyDict(acdefines)
|
|
|
|
@staticmethod
|
|
def from_config_status(path):
|
|
config = BuildConfig.from_config_status(path)
|
|
|
|
return ConfigEnvironment(config.topsrcdir, config.topobjdir,
|
|
config.defines, config.non_global_defines, config.substs, path)
|
|
|
|
|
|
class PartialConfigDict(object):
|
|
"""Facilitates mapping the config.statusd defines & substs with dict-like access.
|
|
|
|
This allows a buildconfig client to use buildconfig.defines['FOO'] (and
|
|
similar for substs), where the value of FOO is delay-loaded until it is
|
|
needed.
|
|
"""
|
|
|
|
def __init__(self, config_statusd, typ, environ_override=False):
|
|
self._dict = {}
|
|
self._datadir = mozpath.join(config_statusd, typ)
|
|
self._config_track = mozpath.join(self._datadir, 'config.track')
|
|
self._files = set()
|
|
self._environ_override = environ_override
|
|
|
|
def _load_config_track(self):
|
|
existing_files = set()
|
|
try:
|
|
with open(self._config_track) as fh:
|
|
existing_files.update(fh.read().splitlines())
|
|
except IOError:
|
|
pass
|
|
return existing_files
|
|
|
|
def _write_file(self, key, value):
|
|
encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
|
|
filename = mozpath.join(self._datadir, key)
|
|
with FileAvoidWrite(filename) as fh:
|
|
json.dump(value, fh, indent=4, encoding=encoding)
|
|
return filename
|
|
|
|
def _fill_group(self, values):
|
|
# Clear out any cached values. This is mostly for tests that will check
|
|
# the environment, write out a new set of variables, and then check the
|
|
# environment again. Normally only configure ends up calling this
|
|
# function, and other consumers create their own
|
|
# PartialConfigEnvironments in new python processes.
|
|
self._dict = {}
|
|
|
|
existing_files = self._load_config_track()
|
|
|
|
new_files = set()
|
|
for k, v in values.iteritems():
|
|
new_files.add(self._write_file(k, v))
|
|
|
|
for filename in existing_files - new_files:
|
|
# We can't actually os.remove() here, since make would not see that the
|
|
# file has been removed and that the target needs to be updated. Instead
|
|
# we just overwrite the file with a value of None, which is equivalent
|
|
# to a non-existing file.
|
|
with FileAvoidWrite(filename) as fh:
|
|
json.dump(None, fh)
|
|
|
|
with FileAvoidWrite(self._config_track) as fh:
|
|
for f in sorted(new_files):
|
|
fh.write('%s\n' % f)
|
|
|
|
def __getitem__(self, key):
|
|
if self._environ_override:
|
|
if (key not in ('CPP', 'CXXCPP', 'SHELL')) and (key in os.environ):
|
|
return os.environ[key]
|
|
|
|
if key not in self._dict:
|
|
data = None
|
|
try:
|
|
filename = mozpath.join(self._datadir, key)
|
|
self._files.add(filename)
|
|
with open(filename) as f:
|
|
data = json.load(f)
|
|
except IOError:
|
|
pass
|
|
self._dict[key] = data
|
|
|
|
if self._dict[key] is None:
|
|
raise KeyError("'%s'" % key)
|
|
return self._dict[key]
|
|
|
|
def __setitem__(self, key, value):
|
|
self._dict[key] = value
|
|
|
|
def get(self, key, default=None):
|
|
return self[key] if key in self else default
|
|
|
|
def __contains__(self, key):
|
|
try:
|
|
return self[key] is not None
|
|
except KeyError:
|
|
return False
|
|
|
|
def iteritems(self):
|
|
existing_files = self._load_config_track()
|
|
for f in existing_files:
|
|
# The track file contains filenames, and the basename is the
|
|
# variable name.
|
|
var = mozpath.basename(f)
|
|
yield var, self[var]
|
|
|
|
|
|
class PartialConfigEnvironment(object):
|
|
"""Allows access to individual config.status items via config.statusd/* files.
|
|
|
|
This class is similar to the full ConfigEnvironment, which uses
|
|
config.status, except this allows access and tracks dependencies to
|
|
individual configure values. It is intended to be used during the build
|
|
process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and
|
|
anything else that may need to access specific substs or defines.
|
|
|
|
Creating a PartialConfigEnvironment requires only the topobjdir, which is
|
|
needed to distinguish between the top-level environment and the js/src
|
|
environment.
|
|
|
|
The PartialConfigEnvironment automatically defines one additional subst variable
|
|
from all the defines not appearing in non_global_defines:
|
|
- ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
|
|
preprocessor command lines. The order in which defines were given
|
|
when creating the ConfigEnvironment is preserved.
|
|
|
|
and one additional define from all the defines as a dictionary:
|
|
- ALLDEFINES contains all of the global defines as a dictionary. This is
|
|
intended to be used instead of the defines structure from config.status so
|
|
that scripts can depend directly on its value.
|
|
"""
|
|
|
|
def __init__(self, topobjdir):
|
|
config_statusd = mozpath.join(topobjdir, 'config.statusd')
|
|
self.substs = PartialConfigDict(config_statusd, 'substs', environ_override=True)
|
|
self.defines = PartialConfigDict(config_statusd, 'defines')
|
|
self.topobjdir = topobjdir
|
|
|
|
def write_vars(self, config):
|
|
substs = config['substs'].copy()
|
|
defines = config['defines'].copy()
|
|
|
|
global_defines = [
|
|
name for name in config['defines']
|
|
if name not in config['non_global_defines']
|
|
]
|
|
acdefines = ' '.join(['-D%s=%s' % (name,
|
|
shell_quote(config['defines'][name]).replace('$', '$$'))
|
|
for name in sorted(global_defines)])
|
|
substs['ACDEFINES'] = acdefines
|
|
|
|
all_defines = OrderedDict()
|
|
for k in global_defines:
|
|
all_defines[k] = config['defines'][k]
|
|
defines['ALLDEFINES'] = all_defines
|
|
|
|
self.substs._fill_group(substs)
|
|
self.defines._fill_group(defines)
|
|
|
|
def get_dependencies(self):
|
|
return ['$(wildcard %s)' % f for f in self.substs._files | self.defines._files]
|