forked from mirrors/gecko-dev
81228f00cd
This was added ing bug 1352595. It has only been used briefly, and we since then have gone all the way to never compress omni.ja. Even if we did go back to compressing, we'd probably go with zstd rather than brotli. The gecko-side support for this has only ever been nightly-only. Differential Revision: https://phabricator.services.mozilla.com/D118655
298 lines
9.2 KiB
Python
298 lines
9.2 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, unicode_literals, print_function
|
|
|
|
from mozpack.packager.formats import (
|
|
FlatFormatter,
|
|
JarFormatter,
|
|
OmniJarFormatter,
|
|
)
|
|
from mozpack.packager import (
|
|
preprocess_manifest,
|
|
preprocess,
|
|
Component,
|
|
SimpleManifestSink,
|
|
)
|
|
from mozpack.files import (
|
|
GeneratedFile,
|
|
FileFinder,
|
|
File,
|
|
)
|
|
from mozpack.copier import (
|
|
FileCopier,
|
|
Jarrer,
|
|
)
|
|
from mozpack.errors import errors
|
|
from mozpack.files import ExecutableFile
|
|
import mozpack.path as mozpath
|
|
import buildconfig
|
|
from argparse import ArgumentParser
|
|
from collections import OrderedDict
|
|
from createprecomplete import generate_precomplete
|
|
import os
|
|
import plistlib
|
|
import six
|
|
from six import StringIO
|
|
import subprocess
|
|
|
|
|
|
class PackagerFileFinder(FileFinder):
|
|
def get(self, path):
|
|
f = super(PackagerFileFinder, self).get(path)
|
|
# Normalize Info.plist files, and remove the MozillaDeveloper*Path
|
|
# entries which are only needed on unpackaged builds.
|
|
if mozpath.basename(path) == "Info.plist":
|
|
info = plistlib.load(f.open(), dict_type=OrderedDict)
|
|
info.pop("MozillaDeveloperObjPath", None)
|
|
info.pop("MozillaDeveloperRepoPath", None)
|
|
return GeneratedFile(plistlib.dumps(info, sort_keys=False))
|
|
return f
|
|
|
|
|
|
class RemovedFiles(GeneratedFile):
|
|
"""
|
|
File class for removed-files. Is used as a preprocessor parser.
|
|
"""
|
|
|
|
def __init__(self, copier):
|
|
self.copier = copier
|
|
GeneratedFile.__init__(self, b"")
|
|
|
|
def handle_line(self, f):
|
|
f = f.strip()
|
|
if not f:
|
|
return
|
|
if self.copier.contains(f):
|
|
errors.error("Removal of packaged file(s): %s" % f)
|
|
self.content += six.ensure_binary(f) + b"\n"
|
|
|
|
|
|
def split_define(define):
|
|
"""
|
|
Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to
|
|
1. Numeric VALs are returned as ints.
|
|
"""
|
|
if "=" in define:
|
|
name, value = define.split("=", 1)
|
|
try:
|
|
value = int(value)
|
|
except ValueError:
|
|
pass
|
|
return (name, value)
|
|
return (define, 1)
|
|
|
|
|
|
class NoPkgFilesRemover(object):
|
|
"""
|
|
Formatter wrapper to handle NO_PKG_FILES.
|
|
"""
|
|
|
|
def __init__(self, formatter, has_manifest):
|
|
assert "NO_PKG_FILES" in os.environ
|
|
self._formatter = formatter
|
|
self._files = os.environ["NO_PKG_FILES"].split()
|
|
if has_manifest:
|
|
self._error = errors.error
|
|
self._msg = "NO_PKG_FILES contains file listed in manifest: %s"
|
|
else:
|
|
self._error = errors.warn
|
|
self._msg = "Skipping %s"
|
|
|
|
def add_base(self, base, *args):
|
|
self._formatter.add_base(base, *args)
|
|
|
|
def add(self, path, content):
|
|
if not any(mozpath.match(path, spec) for spec in self._files):
|
|
self._formatter.add(path, content)
|
|
else:
|
|
self._error(self._msg % path)
|
|
|
|
def add_manifest(self, entry):
|
|
self._formatter.add_manifest(entry)
|
|
|
|
def contains(self, path):
|
|
return self._formatter.contains(path)
|
|
|
|
|
|
def main():
|
|
parser = ArgumentParser()
|
|
parser.add_argument(
|
|
"-D",
|
|
dest="defines",
|
|
action="append",
|
|
metavar="VAR[=VAL]",
|
|
help="Define a variable",
|
|
)
|
|
parser.add_argument(
|
|
"--format",
|
|
default="omni",
|
|
help="Choose the chrome format for packaging "
|
|
+ "(omni, jar or flat ; default: %(default)s)",
|
|
)
|
|
parser.add_argument("--removals", default=None, help="removed-files source file")
|
|
parser.add_argument(
|
|
"--ignore-errors",
|
|
action="store_true",
|
|
default=False,
|
|
help="Transform errors into warnings.",
|
|
)
|
|
parser.add_argument(
|
|
"--ignore-broken-symlinks",
|
|
action="store_true",
|
|
default=False,
|
|
help="Do not fail when processing broken symlinks.",
|
|
)
|
|
parser.add_argument(
|
|
"--minify",
|
|
action="store_true",
|
|
default=False,
|
|
help="Make some files more compact while packaging",
|
|
)
|
|
parser.add_argument(
|
|
"--minify-js",
|
|
action="store_true",
|
|
help="Minify JavaScript files while packaging.",
|
|
)
|
|
parser.add_argument(
|
|
"--js-binary",
|
|
help="Path to js binary. This is used to verify "
|
|
"minified JavaScript. If this is not defined, "
|
|
"minification verification will not be performed.",
|
|
)
|
|
parser.add_argument(
|
|
"--jarlog", default="", help="File containing jar " + "access logs"
|
|
)
|
|
parser.add_argument(
|
|
"--compress",
|
|
choices=("none", "deflate"),
|
|
default="deflate",
|
|
help="Use given jar compression (default: deflate)",
|
|
)
|
|
parser.add_argument("manifest", default=None, nargs="?", help="Manifest file name")
|
|
parser.add_argument("source", help="Source directory")
|
|
parser.add_argument("destination", help="Destination directory")
|
|
parser.add_argument(
|
|
"--non-resource",
|
|
nargs="+",
|
|
metavar="PATTERN",
|
|
default=[],
|
|
help="Extra files not to be considered as resources",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
defines = dict(buildconfig.defines["ALLDEFINES"])
|
|
if args.ignore_errors:
|
|
errors.ignore_errors()
|
|
|
|
if args.defines:
|
|
for name, value in [split_define(d) for d in args.defines]:
|
|
defines[name] = value
|
|
|
|
compress = {
|
|
"none": False,
|
|
"deflate": True,
|
|
}[args.compress]
|
|
|
|
copier = FileCopier()
|
|
if args.format == "flat":
|
|
formatter = FlatFormatter(copier)
|
|
elif args.format == "jar":
|
|
formatter = JarFormatter(copier, compress=compress)
|
|
elif args.format == "omni":
|
|
formatter = OmniJarFormatter(
|
|
copier,
|
|
buildconfig.substs["OMNIJAR_NAME"],
|
|
compress=compress,
|
|
non_resources=args.non_resource,
|
|
)
|
|
else:
|
|
errors.fatal("Unknown format: %s" % args.format)
|
|
|
|
# Adjust defines according to the requested format.
|
|
if isinstance(formatter, OmniJarFormatter):
|
|
defines["MOZ_OMNIJAR"] = 1
|
|
elif "MOZ_OMNIJAR" in defines:
|
|
del defines["MOZ_OMNIJAR"]
|
|
|
|
respath = ""
|
|
if "RESPATH" in defines:
|
|
respath = SimpleManifestSink.normalize_path(defines["RESPATH"])
|
|
while respath.startswith("/"):
|
|
respath = respath[1:]
|
|
|
|
with errors.accumulate():
|
|
finder_args = dict(
|
|
minify=args.minify,
|
|
minify_js=args.minify_js,
|
|
ignore_broken_symlinks=args.ignore_broken_symlinks,
|
|
)
|
|
if args.js_binary:
|
|
finder_args["minify_js_verify_command"] = [
|
|
args.js_binary,
|
|
os.path.join(
|
|
os.path.abspath(os.path.dirname(__file__)), "js-compare-ast.js"
|
|
),
|
|
]
|
|
finder = PackagerFileFinder(args.source, find_executables=True, **finder_args)
|
|
if "NO_PKG_FILES" in os.environ:
|
|
sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None)
|
|
else:
|
|
sinkformatter = formatter
|
|
sink = SimpleManifestSink(finder, sinkformatter)
|
|
if args.manifest:
|
|
preprocess_manifest(sink, args.manifest, defines)
|
|
else:
|
|
sink.add(Component(""), "bin/*")
|
|
sink.close(args.manifest is not None)
|
|
|
|
if args.removals:
|
|
removals_in = StringIO(open(args.removals).read())
|
|
removals_in.name = args.removals
|
|
removals = RemovedFiles(copier)
|
|
preprocess(removals_in, removals, defines)
|
|
copier.add(mozpath.join(respath, "removed-files"), removals)
|
|
|
|
# If a pdb file is present and we were instructed to copy it, include it.
|
|
# Run on all OSes to capture MinGW builds
|
|
if buildconfig.substs.get("MOZ_COPY_PDBS"):
|
|
# We want to mutate the copier while we're iterating through it, so copy
|
|
# the items to a list first.
|
|
copier_items = [(p, f) for p, f in copier]
|
|
for p, f in copier_items:
|
|
if isinstance(f, ExecutableFile):
|
|
pdbname = os.path.splitext(f.inputs()[0])[0] + ".pdb"
|
|
if os.path.exists(pdbname):
|
|
copier.add(os.path.basename(pdbname), File(pdbname))
|
|
|
|
# Setup preloading
|
|
if args.jarlog:
|
|
if not os.path.exists(args.jarlog):
|
|
raise Exception("Cannot find jar log: %s" % args.jarlog)
|
|
omnijars = []
|
|
if isinstance(formatter, OmniJarFormatter):
|
|
omnijars = [
|
|
mozpath.join(base, buildconfig.substs["OMNIJAR_NAME"])
|
|
for base in sink.packager.get_bases(addons=False)
|
|
]
|
|
|
|
from mozpack.mozjar import JarLog
|
|
|
|
log = JarLog(args.jarlog)
|
|
for p, f in copier:
|
|
if not isinstance(f, Jarrer):
|
|
continue
|
|
if respath:
|
|
p = mozpath.relpath(p, respath)
|
|
if p in log:
|
|
f.preload(log[p])
|
|
elif p in omnijars:
|
|
raise Exception("No jar log data for %s" % p)
|
|
|
|
copier.copy(args.destination)
|
|
generate_precomplete(os.path.normpath(os.path.join(args.destination, respath)))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|