forked from mirrors/gecko-dev
843a1b6c03
By using environment variables, the earlier _HAS_EXCEPTIONS hack can be removed. Differential Revision: https://phabricator.services.mozilla.com/D152115
467 lines
16 KiB
Python
Executable file
467 lines
16 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# ***** BEGIN LICENSE BLOCK *****
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
# ***** END LICENSE BLOCK *****
|
|
from __future__ import absolute_import
|
|
import sys
|
|
import os
|
|
import glob
|
|
import re
|
|
import subprocess
|
|
|
|
# load modules from parent dir
|
|
sys.path.insert(1, os.path.dirname(sys.path[0]))
|
|
|
|
# import the guts
|
|
import mozharness
|
|
from mozharness.base.vcs.vcsbase import VCSScript
|
|
from mozharness.base.log import ERROR, DEBUG, FATAL
|
|
from mozharness.base.transfer import TransferMixin
|
|
from mozharness.mozilla.tooltool import TooltoolMixin
|
|
|
|
|
|
external_tools_path = os.path.join(
|
|
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
|
|
"external_tools",
|
|
)
|
|
|
|
|
|
class OpenH264Build(TransferMixin, VCSScript, TooltoolMixin):
|
|
all_actions = [
|
|
"clobber",
|
|
"get-tooltool",
|
|
"checkout-sources",
|
|
"build",
|
|
"test",
|
|
"package",
|
|
"dump-symbols",
|
|
]
|
|
|
|
default_actions = [
|
|
"get-tooltool",
|
|
"checkout-sources",
|
|
"build",
|
|
"package",
|
|
"dump-symbols",
|
|
]
|
|
|
|
config_options = [
|
|
[
|
|
["--repo"],
|
|
{
|
|
"dest": "repo",
|
|
"help": "OpenH264 repository to use",
|
|
"default": "https://github.com/dminor/openh264.git",
|
|
},
|
|
],
|
|
[
|
|
["--rev"],
|
|
{"dest": "revision", "help": "revision to checkout", "default": "master"},
|
|
],
|
|
[
|
|
["--debug"],
|
|
{
|
|
"dest": "debug_build",
|
|
"action": "store_true",
|
|
"help": "Do a debug build",
|
|
},
|
|
],
|
|
[
|
|
["--arch"],
|
|
{
|
|
"dest": "arch",
|
|
"help": "Arch type to use (x64, x86, arm, or aarch64)",
|
|
},
|
|
],
|
|
[
|
|
["--os"],
|
|
{
|
|
"dest": "operating_system",
|
|
"help": "Specify the operating system to build for",
|
|
},
|
|
],
|
|
[
|
|
["--branch"],
|
|
{
|
|
"dest": "branch",
|
|
"help": "dummy option",
|
|
},
|
|
],
|
|
[
|
|
["--build-pool"],
|
|
{
|
|
"dest": "build_pool",
|
|
"help": "dummy option",
|
|
},
|
|
],
|
|
]
|
|
|
|
def __init__(
|
|
self,
|
|
require_config_file=False,
|
|
config={},
|
|
all_actions=all_actions,
|
|
default_actions=default_actions,
|
|
):
|
|
|
|
# Default configuration
|
|
default_config = {
|
|
"debug_build": False,
|
|
"upload_ssh_key": "~/.ssh/ffxbld_rsa",
|
|
"upload_ssh_user": "ffxbld",
|
|
"upload_ssh_host": "upload.ffxbld.productdelivery.prod.mozaws.net",
|
|
"upload_path_base": "/tmp/openh264",
|
|
}
|
|
default_config.update(config)
|
|
|
|
VCSScript.__init__(
|
|
self,
|
|
config_options=self.config_options,
|
|
require_config_file=require_config_file,
|
|
config=default_config,
|
|
all_actions=all_actions,
|
|
default_actions=default_actions,
|
|
)
|
|
|
|
def query_abs_dirs(self):
|
|
if self.abs_dirs:
|
|
return self.abs_dirs
|
|
dirs = super(OpenH264Build, self).query_abs_dirs()
|
|
dirs["abs_upload_dir"] = os.path.join(dirs["abs_work_dir"], "upload")
|
|
self.abs_dirs = dirs
|
|
return self.abs_dirs
|
|
|
|
def get_tooltool(self):
|
|
c = self.config
|
|
if not c.get("tooltool_manifest_file"):
|
|
self.info("Skipping tooltool fetching since no tooltool manifest")
|
|
return
|
|
dirs = self.query_abs_dirs()
|
|
self.mkdir_p(dirs["abs_work_dir"])
|
|
manifest = os.path.join(
|
|
dirs["abs_src_dir"],
|
|
"testing",
|
|
"mozharness",
|
|
"configs",
|
|
"openh264",
|
|
"tooltool-manifests",
|
|
c["tooltool_manifest_file"],
|
|
)
|
|
self.info("Getting tooltool files from manifest (%s)" % manifest)
|
|
try:
|
|
self.tooltool_fetch(
|
|
manifest=manifest,
|
|
output_dir=os.path.join(dirs["abs_work_dir"]),
|
|
cache=c.get("tooltool_cache"),
|
|
)
|
|
except KeyError:
|
|
self.error("missing a required key.")
|
|
|
|
def query_package_name(self):
|
|
if self.config["arch"] in ("x64", "aarch64"):
|
|
bits = "64"
|
|
else:
|
|
bits = "32"
|
|
version = self.config["revision"]
|
|
|
|
if sys.platform in ("linux2", "linux"):
|
|
if self.config.get("operating_system") == "android":
|
|
return "openh264-android-{arch}-{version}.zip".format(
|
|
version=version, arch=self.config["arch"]
|
|
)
|
|
elif self.config.get("operating_system") == "darwin":
|
|
suffix = ""
|
|
if self.config["arch"] != "x64":
|
|
suffix = "-" + self.config["arch"]
|
|
return "openh264-macosx{bits}{suffix}-{version}.zip".format(
|
|
version=version, bits=bits, suffix=suffix
|
|
)
|
|
else:
|
|
return "openh264-linux{bits}-{version}.zip".format(
|
|
version=version, bits=bits
|
|
)
|
|
elif sys.platform == "win32":
|
|
if self.config["arch"] == "aarch64":
|
|
return "openh264-win64-aarch64-{version}.zip".format(version=version)
|
|
else:
|
|
return "openh264-win{bits}-{version}.zip".format(
|
|
version=version, bits=bits
|
|
)
|
|
self.fatal("can't determine platform")
|
|
|
|
def query_make_params(self):
|
|
retval = []
|
|
if self.config["debug_build"]:
|
|
retval.append("BUILDTYPE=Debug")
|
|
|
|
if self.config["arch"] in ("x64", "aarch64"):
|
|
retval.append("ENABLE64BIT=Yes")
|
|
else:
|
|
retval.append("ENABLE64BIT=No")
|
|
|
|
if self.config["arch"] == "x86":
|
|
retval.append("ARCH=x86")
|
|
elif self.config["arch"] == "x64":
|
|
retval.append("ARCH=x86_64")
|
|
elif self.config["arch"] == "aarch64":
|
|
retval.append("ARCH=arm64")
|
|
else:
|
|
self.fatal("Unknown arch: {}".format(self.config["arch"]))
|
|
|
|
if "operating_system" in self.config:
|
|
retval.append("OS=%s" % self.config["operating_system"])
|
|
if self.config["operating_system"] == "android":
|
|
retval.append("TARGET=invalid")
|
|
retval.append("NDKLEVEL=%s" % self.config["min_sdk"])
|
|
retval.append("NDKROOT=%s/android-ndk" % os.environ["MOZ_FETCHES_DIR"])
|
|
retval.append("NDK_TOOLCHAIN_VERSION=clang")
|
|
if self.config["operating_system"] == "darwin":
|
|
retval.append("OS=darwin")
|
|
|
|
if self._is_windows():
|
|
retval.append("OS=msvc")
|
|
retval.append("CC=clang-cl")
|
|
retval.append("CXX=clang-cl")
|
|
if self.config["arch"] == "aarch64":
|
|
retval.append("CXX_LINK_O=-nologo --target=aarch64-windows-msvc -Fe$@")
|
|
else:
|
|
retval.append("CC=clang")
|
|
retval.append("CXX=clang++")
|
|
|
|
return retval
|
|
|
|
def query_upload_ssh_key(self):
|
|
return self.config["upload_ssh_key"]
|
|
|
|
def query_upload_ssh_host(self):
|
|
return self.config["upload_ssh_host"]
|
|
|
|
def query_upload_ssh_user(self):
|
|
return self.config["upload_ssh_user"]
|
|
|
|
def query_upload_ssh_path(self):
|
|
return "%s/%s" % (self.config["upload_path_base"], self.config["revision"])
|
|
|
|
def run_make(self, target, capture_output=False):
|
|
cmd = ["make", target] + self.query_make_params()
|
|
dirs = self.query_abs_dirs()
|
|
repo_dir = os.path.join(dirs["abs_work_dir"], "openh264")
|
|
env = None
|
|
if self.config.get("partial_env"):
|
|
env = self.query_env(self.config["partial_env"])
|
|
kwargs = dict(cwd=repo_dir, env=env)
|
|
if capture_output:
|
|
return self.get_output_from_command(cmd, **kwargs)
|
|
else:
|
|
return self.run_command(cmd, **kwargs)
|
|
|
|
def _git_checkout(self, repo, repo_dir, rev):
|
|
try:
|
|
subprocess.run(["git", "clone", "-q", "--no-checkout", repo, repo_dir])
|
|
subprocess.run(["git", "checkout", "-q", "-f", f"{rev}^0"], cwd=repo_dir)
|
|
except Exception:
|
|
self.rmtree(repo_dir)
|
|
raise
|
|
return True
|
|
|
|
def checkout_sources(self):
|
|
repo = self.config["repo"]
|
|
rev = self.config["revision"]
|
|
|
|
dirs = self.query_abs_dirs()
|
|
repo_dir = os.path.join(dirs["abs_work_dir"], "openh264")
|
|
|
|
if self._is_windows():
|
|
# We don't have git on our windows builders, so download a zip
|
|
# package instead.
|
|
path = repo.replace(".git", "/archive/") + rev + ".zip"
|
|
self.download_file(path)
|
|
self.unzip(rev + ".zip", dirs["abs_work_dir"])
|
|
self.move(
|
|
os.path.join(dirs["abs_work_dir"], "openh264-" + rev),
|
|
os.path.join(dirs["abs_work_dir"], "openh264"),
|
|
)
|
|
|
|
# Retrieve in-tree version of gmp-api
|
|
self.copytree(
|
|
os.path.join(dirs["abs_src_dir"], "dom", "media", "gmp", "gmp-api"),
|
|
os.path.join(repo_dir, "gmp-api"),
|
|
)
|
|
|
|
# We need gas-preprocessor.pl for arm64 builds
|
|
if self.config["arch"] == "aarch64":
|
|
openh264_dir = os.path.join(dirs["abs_work_dir"], "openh264")
|
|
self.download_file(
|
|
(
|
|
"https://raw.githubusercontent.com/libav/"
|
|
"gas-preprocessor/c2bc63c96678d9739509e58"
|
|
"7aa30c94bdc0e636d/gas-preprocessor.pl"
|
|
),
|
|
parent_dir=openh264_dir,
|
|
)
|
|
self.chmod(os.path.join(openh264_dir, "gas-preprocessor.pl"), 744)
|
|
|
|
# gas-preprocessor.pl expects cpp to exist
|
|
# os.symlink is not available on Windows until we switch to
|
|
# Python 3.
|
|
os.system(
|
|
"ln -s %s %s"
|
|
% (
|
|
os.path.join(
|
|
os.environ["MOZ_FETCHES_DIR"], "clang", "bin", "clang.exe"
|
|
),
|
|
os.path.join(openh264_dir, "cpp"),
|
|
)
|
|
)
|
|
return 0
|
|
|
|
self.retry(
|
|
self._git_checkout,
|
|
error_level=FATAL,
|
|
error_message="Automation Error: couldn't clone repo",
|
|
args=(repo, repo_dir, rev),
|
|
)
|
|
|
|
# Checkout gmp-api
|
|
# TODO: Nothing here updates it yet, or enforces versions!
|
|
if not os.path.exists(os.path.join(repo_dir, "gmp-api")):
|
|
retval = self.run_make("gmp-bootstrap")
|
|
if retval != 0:
|
|
self.fatal("couldn't bootstrap gmp")
|
|
else:
|
|
self.info("skipping gmp bootstrap - we have it locally")
|
|
|
|
# Checkout gtest
|
|
# TODO: Requires svn!
|
|
if not os.path.exists(os.path.join(repo_dir, "gtest")):
|
|
retval = self.run_make("gtest-bootstrap")
|
|
if retval != 0:
|
|
self.fatal("couldn't bootstrap gtest")
|
|
else:
|
|
self.info("skipping gtest bootstrap - we have it locally")
|
|
|
|
return retval
|
|
|
|
def build(self):
|
|
retval = self.run_make("plugin")
|
|
if retval != 0:
|
|
self.fatal("couldn't build plugin")
|
|
|
|
def package(self):
|
|
dirs = self.query_abs_dirs()
|
|
srcdir = os.path.join(dirs["abs_work_dir"], "openh264")
|
|
package_name = self.query_package_name()
|
|
package_file = os.path.join(dirs["abs_work_dir"], package_name)
|
|
if os.path.exists(package_file):
|
|
os.unlink(package_file)
|
|
to_package = []
|
|
for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*")):
|
|
if not re.search(
|
|
"(?:lib)?gmpopenh264(?!\.\d)\.(?:dylib|so|dll|info)(?!\.\d)", f
|
|
):
|
|
# Don't package unnecessary zip bloat
|
|
# Blocks things like libgmpopenh264.2.dylib and libgmpopenh264.so.1
|
|
self.log("Skipping packaging of {package}".format(package=f))
|
|
continue
|
|
to_package.append(os.path.basename(f))
|
|
self.log("Packaging files %s" % to_package)
|
|
cmd = ["zip", package_file] + to_package
|
|
retval = self.run_command(cmd, cwd=srcdir)
|
|
if retval != 0:
|
|
self.fatal("couldn't make package")
|
|
self.copy_to_upload_dir(
|
|
package_file, dest=os.path.join(srcdir, "artifacts", package_name)
|
|
)
|
|
|
|
# Taskcluster expects this path to exist, but we don't use it
|
|
# because our builds are private.
|
|
path = os.path.join(
|
|
self.query_abs_dirs()["abs_work_dir"], "..", "public", "build"
|
|
)
|
|
self.mkdir_p(path)
|
|
|
|
def dump_symbols(self):
|
|
dirs = self.query_abs_dirs()
|
|
c = self.config
|
|
srcdir = os.path.join(dirs["abs_work_dir"], "openh264")
|
|
package_name = self.run_make("echo-plugin-name", capture_output=True)
|
|
if not package_name:
|
|
self.fatal("failure running make")
|
|
zip_package_name = self.query_package_name()
|
|
if not zip_package_name[-4:] == ".zip":
|
|
self.fatal("Unexpected zip_package_name")
|
|
symbol_package_name = "{base}.symbols.zip".format(base=zip_package_name[:-4])
|
|
symbol_zip_path = os.path.join(srcdir, "artifacts", symbol_package_name)
|
|
repo_dir = os.path.join(dirs["abs_work_dir"], "openh264")
|
|
env = None
|
|
if self.config.get("partial_env"):
|
|
env = self.query_env(self.config["partial_env"])
|
|
kwargs = dict(cwd=repo_dir, env=env)
|
|
dump_syms = os.path.join(dirs["abs_work_dir"], c["dump_syms_binary"])
|
|
self.chmod(dump_syms, 0o755)
|
|
python = self.query_exe("python3")
|
|
cmd = [
|
|
python,
|
|
os.path.join(external_tools_path, "packagesymbols.py"),
|
|
"--symbol-zip",
|
|
symbol_zip_path,
|
|
dump_syms,
|
|
os.path.join(srcdir, package_name),
|
|
]
|
|
self.run_command(cmd, **kwargs)
|
|
|
|
def test(self):
|
|
retval = self.run_make("test")
|
|
if retval != 0:
|
|
self.fatal("test failures")
|
|
|
|
def copy_to_upload_dir(
|
|
self,
|
|
target,
|
|
dest=None,
|
|
log_level=DEBUG,
|
|
error_level=ERROR,
|
|
compress=False,
|
|
upload_dir=None,
|
|
):
|
|
"""Copy target file to upload_dir/dest.
|
|
|
|
Potentially update a manifest in the future if we go that route.
|
|
|
|
Currently only copies a single file; would be nice to allow for
|
|
recursive copying; that would probably done by creating a helper
|
|
_copy_file_to_upload_dir().
|
|
"""
|
|
dest_filename_given = dest is not None
|
|
if upload_dir is None:
|
|
upload_dir = self.query_abs_dirs()["abs_upload_dir"]
|
|
if dest is None:
|
|
dest = os.path.basename(target)
|
|
if dest.endswith("/"):
|
|
dest_file = os.path.basename(target)
|
|
dest_dir = os.path.join(upload_dir, dest)
|
|
dest_filename_given = False
|
|
else:
|
|
dest_file = os.path.basename(dest)
|
|
dest_dir = os.path.join(upload_dir, os.path.dirname(dest))
|
|
if compress and not dest_filename_given:
|
|
dest_file += ".gz"
|
|
dest = os.path.join(dest_dir, dest_file)
|
|
if not os.path.exists(target):
|
|
self.log("%s doesn't exist!" % target, level=error_level)
|
|
return None
|
|
self.mkdir_p(dest_dir)
|
|
self.copyfile(target, dest, log_level=log_level, compress=compress)
|
|
if os.path.exists(dest):
|
|
return dest
|
|
else:
|
|
self.log("%s doesn't exist after copy!" % dest, level=error_level)
|
|
return None
|
|
|
|
|
|
# main {{{1
|
|
if __name__ == "__main__":
|
|
myScript = OpenH264Build()
|
|
myScript.run_and_exit()
|