gecko-dev/testing/mochitest/runrobocop.py
Michael Comella 39ed656f3a Bug 1261494 - Reduce telemetry init delay to 1 second for integration testing. r=gbrown
My one concern is that this change could increase the amount of processing
time spent on telemetry initialization, causing the runtime of the robocop
test suite to increase. Checking my try push [1] against other try pushes,
it doesn't seem to have made a significant difference, but the change
in runtime between pushes can be large (e.g. > 5min) so it's hard to
tell.

[1]: https://treeherder.mozilla.org/#/jobs?repo=try&revision=2017843315fe&selectedJob=24641374

MozReview-Commit-ID: LeeGgNEp74h

--HG--
extra : rebase_source : 21b01fa8a5357de19046fc946b4098cfd0f7b823
extra : amend_source : 457f229e6b92b8834ddd6dfef5837753f47d570b
2016-07-27 08:05:12 -07:00

588 lines
24 KiB
Python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import shutil
import sys
import tempfile
import traceback
sys.path.insert(
0, os.path.abspath(
os.path.realpath(
os.path.dirname(__file__))))
from automation import Automation
from remoteautomation import RemoteAutomation, fennecLogcatFilters
from runtests import KeyValueParseError, MochitestDesktop, MessageLogger, parseKeyValue
from mochitest_options import MochitestArgumentParser
from manifestparser import TestManifest
from manifestparser.filters import chunk_by_slice
import mozdevice
import mozinfo
SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
# TODO inherit from MochitestBase instead
class RobocopTestRunner(MochitestDesktop):
"""
A test harness for Robocop. Robocop tests are UI tests for Firefox for Android,
based on the Robotium test framework. This harness leverages some functionality
from mochitest, for convenience.
"""
auto = None
dm = None
# Some robocop tests run for >60 seconds without generating any output.
NO_OUTPUT_TIMEOUT = 180
def __init__(self, automation, devmgr, options):
"""
Simple one-time initialization.
"""
MochitestDesktop.__init__(self, options)
self.auto = automation
self.dm = devmgr
self.dm.default_timeout = 320
self.options = options
self.options.logFile = "robocop.log"
self.environment = self.auto.environment
self.deviceRoot = self.dm.getDeviceRoot()
self.remoteProfile = options.remoteTestRoot + "/profile"
self.remoteProfileCopy = options.remoteTestRoot + "/profile-copy"
self.auto.setRemoteProfile(self.remoteProfile)
self.remoteConfigFile = os.path.join(
self.deviceRoot, "robotium.config")
self.remoteLog = options.remoteLogFile
self.auto.setRemoteLog(self.remoteLog)
self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots"
self.remoteMozLog = os.path.join(options.remoteTestRoot, "mozlog")
self.auto.setServerInfo(
self.options.webServer, self.options.httpPort, self.options.sslPort)
self.localLog = options.logFile
self.localProfile = None
productPieces = self.options.remoteProductName.split('.')
if (productPieces is not None):
self.auto.setProduct(productPieces[0])
else:
self.auto.setProduct(self.options.remoteProductName)
self.auto.setAppName(self.options.remoteappname)
self.certdbNew = True
self.remoteCopyAvailable = True
self.passed = 0
self.failed = 0
self.todo = 0
def startup(self):
"""
Second-stage initialization: One-time initialization which may require cleanup.
"""
# Despite our efforts to clean up servers started by this script, in practice
# we still see infrequent cases where a process is orphaned and interferes
# with future tests, typically because the old server is keeping the port in use.
# Try to avoid those failures by checking for and killing orphan servers before
# trying to start new ones.
self.killNamedOrphans('ssltunnel')
self.killNamedOrphans('xpcshell')
self.auto.deleteANRs()
self.auto.deleteTombstones()
self.dm.killProcess(self.options.app.split('/')[-1])
self.dm.removeDir(self.remoteScreenshots)
self.dm.removeDir(self.remoteMozLog)
self.dm.mkDir(self.remoteMozLog)
self.dm.mkDir(os.path.dirname(self.options.remoteLogFile))
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
androidVersion = self.dm.shellCheckOutput(
['getprop', 'ro.build.version.sdk'])
self.log.info(
"Android sdk version '%s'; will use this to filter manifests" %
str(androidVersion))
mozinfo.info['android_version'] = androidVersion
if (self.options.dm_trans == 'adb' and self.options.robocopApk):
self.dm._checkCmd(["install", "-r", self.options.robocopApk])
self.log.debug("Robocop APK %s installed" %
self.options.robocopApk)
# Display remote diagnostics; if running in mach, keep output terse.
if self.options.log_mach is None:
self.printDeviceInfo()
self.setupLocalPaths()
self.buildProfile()
# ignoreSSLTunnelExts is a workaround for bug 1109310
self.startServers(
self.options,
debuggerInfo=None,
ignoreSSLTunnelExts=True)
self.log.debug("Servers started")
def cleanup(self):
"""
Cleanup at end of job run.
"""
self.log.debug("Cleaning up...")
self.stopServers()
self.dm.killProcess(self.options.app.split('/')[-1])
blobberUploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
if blobberUploadDir:
self.log.debug("Pulling any remote moz logs and screenshots to %s." %
blobberUploadDir)
self.dm.getDirectory(self.remoteMozLog, blobberUploadDir)
self.dm.getDirectory(self.remoteScreenshots, blobberUploadDir)
MochitestDesktop.cleanup(self, self.options)
if self.localProfile:
os.system("rm -Rf %s" % self.localProfile)
self.dm.removeDir(self.remoteProfile)
self.dm.removeDir(self.remoteProfileCopy)
self.dm.removeDir(self.remoteScreenshots)
self.dm.removeDir(self.remoteMozLog)
self.dm.removeFile(self.remoteConfigFile)
if self.dm.fileExists(self.remoteLog):
self.dm.removeFile(self.remoteLog)
self.log.debug("Cleanup complete.")
def findPath(self, paths, filename=None):
for path in paths:
p = path
if filename:
p = os.path.join(p, filename)
if os.path.exists(self.getFullPath(p)):
return path
return None
def makeLocalAutomation(self):
localAutomation = Automation()
localAutomation.IS_WIN32 = False
localAutomation.IS_LINUX = False
localAutomation.IS_MAC = False
localAutomation.UNIXISH = False
hostos = sys.platform
if (hostos == 'mac' or hostos == 'darwin'):
localAutomation.IS_MAC = True
elif (hostos == 'linux' or hostos == 'linux2'):
localAutomation.IS_LINUX = True
localAutomation.UNIXISH = True
elif (hostos == 'win32' or hostos == 'win64'):
localAutomation.BIN_SUFFIX = ".exe"
localAutomation.IS_WIN32 = True
return localAutomation
def setupLocalPaths(self):
"""
Setup xrePath and utilityPath and verify xpcshell.
This is similar to switchToLocalPaths in runtestsremote.py.
"""
localAutomation = self.makeLocalAutomation()
paths = [
self.options.xrePath,
localAutomation.DIST_BIN,
self.auto._product,
os.path.join('..', self.auto._product)
]
self.options.xrePath = self.findPath(paths)
if self.options.xrePath is None:
self.log.error(
"unable to find xulrunner path for %s, please specify with --xre-path" %
os.name)
sys.exit(1)
self.log.debug("using xre path %s" % self.options.xrePath)
xpcshell = "xpcshell"
if (os.name == "nt"):
xpcshell += ".exe"
if self.options.utilityPath:
paths = [self.options.utilityPath, self.options.xrePath]
else:
paths = [self.options.xrePath]
self.options.utilityPath = self.findPath(paths, xpcshell)
if self.options.utilityPath is None:
self.log.error(
"unable to find utility path for %s, please specify with --utility-path" %
os.name)
sys.exit(1)
self.log.debug("using utility path %s" % self.options.utilityPath)
xpcshell_path = os.path.join(self.options.utilityPath, xpcshell)
if localAutomation.elf_arm(xpcshell_path):
self.log.error('xpcshell at %s is an ARM binary; please use '
'the --utility-path argument to specify the path '
'to a desktop version.' % xpcshell_path)
sys.exit(1)
self.log.debug("xpcshell found at %s" % xpcshell_path)
def buildProfile(self):
"""
Build a profile locally, keep it locally for use by servers and
push a copy to the remote profile-copy directory.
This is similar to buildProfile in runtestsremote.py.
"""
self.options.extraPrefs.append('browser.search.suggest.enabled=true')
self.options.extraPrefs.append('browser.search.suggest.prompted=true')
self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
self.options.extraPrefs.append('browser.snippets.enabled=false')
self.options.extraPrefs.append('browser.casting.enabled=true')
self.options.extraPrefs.append('extensions.autoupdate.enabled=false')
# Override the telemetry init delay for integration testing.
self.options.extraPrefs.append('toolkit.telemetry.initDelay=1')
self.options.extensionsToExclude.extend([
'mochikit@mozilla.org',
'worker-test@mozilla.org.xpi',
'workerbootstrap-test@mozilla.org.xpi',
'indexedDB-test@mozilla.org.xpi',
])
manifest = MochitestDesktop.buildProfile(self, self.options)
self.localProfile = self.options.profilePath
self.log.debug("Profile created at %s" % self.localProfile)
# some files are not needed for robocop; save time by not pushing
shutil.rmtree(os.path.join(self.localProfile, 'webapps'))
os.remove(os.path.join(self.localProfile, 'userChrome.css'))
try:
self.dm.pushDir(self.localProfile, self.remoteProfileCopy)
except mozdevice.DMError:
self.log.error(
"Automation Error: Unable to copy profile to device.")
raise
return manifest
def setupRemoteProfile(self):
"""
Remove any remote profile and re-create it.
"""
self.log.debug("Updating remote profile at %s" % self.remoteProfile)
self.dm.removeDir(self.remoteProfile)
if self.remoteCopyAvailable:
try:
self.dm.shellCheckOutput(
['cp', '-r', self.remoteProfileCopy, self.remoteProfile],
root=True, timeout=60)
except mozdevice.DMError:
# For instance, cp is not available on some older versions of
# Android.
self.log.info(
"Unable to copy remote profile; falling back to push.")
self.remoteCopyAvailable = False
if not self.remoteCopyAvailable:
self.dm.pushDir(self.localProfile, self.remoteProfile)
def parseLocalLog(self):
"""
Read and parse the local log file, noting any failures.
"""
with open(self.localLog) as currentLog:
data = currentLog.readlines()
os.unlink(self.localLog)
start_found = False
end_found = False
fail_found = False
for line in data:
try:
message = json.loads(line)
if not isinstance(message, dict) or 'action' not in message:
continue
except ValueError:
continue
if message['action'] == 'test_end':
end_found = True
start_found = False
break
if start_found and not end_found:
if 'status' in message:
if 'expected' in message:
self.failed += 1
elif message['status'] == 'PASS':
self.passed += 1
elif message['status'] == 'FAIL':
self.todo += 1
if message['action'] == 'test_start':
start_found = True
if 'expected' in message:
fail_found = True
result = 0
if fail_found:
result = 1
if not end_found:
self.log.info(
"PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)")
result = 1
return result
def logTestSummary(self):
"""
Print a summary of all tests run to stdout, for treeherder parsing
(logging via self.log does not work here).
"""
print("0 INFO TEST-START | Shutdown")
print("1 INFO Passed: %s" % (self.passed))
print("2 INFO Failed: %s" % (self.failed))
print("3 INFO Todo: %s" % (self.todo))
print("4 INFO SimpleTest FINISHED")
if self.failed > 0:
return 1
return 0
def printDeviceInfo(self, printLogcat=False):
"""
Log remote device information and logcat (if requested).
This is similar to printDeviceInfo in runtestsremote.py
"""
try:
if printLogcat:
logcat = self.dm.getLogcat(
filterOutRegexps=fennecLogcatFilters)
self.log.info(
'\n' +
''.join(logcat).decode(
'utf-8',
'replace'))
self.log.info("Device info:")
devinfo = self.dm.getInfo()
for category in devinfo:
if type(devinfo[category]) is list:
self.log.info(" %s:" % category)
for item in devinfo[category]:
self.log.info(" %s" % item)
else:
self.log.info(" %s: %s" % (category, devinfo[category]))
self.log.info("Test root: %s" % self.dm.deviceRoot)
except mozdevice.DMError:
self.log.warning("Error getting device information")
def setupRobotiumConfig(self, browserEnv):
"""
Create robotium.config and push it to the device.
"""
fHandle = tempfile.NamedTemporaryFile(suffix='.config',
prefix='robotium-',
dir=os.getcwd(),
delete=False)
fHandle.write("profile=%s\n" % (self.remoteProfile))
fHandle.write("logfile=%s\n" % (self.options.remoteLogFile))
fHandle.write("host=http://mochi.test:8888/tests\n")
fHandle.write(
"rawhost=http://%s:%s/tests\n" %
(self.options.remoteWebServer, self.options.httpPort))
if browserEnv:
envstr = ""
delim = ""
for key, value in browserEnv.items():
try:
value.index(',')
self.log.error(
"setupRobotiumConfig: browserEnv - Found a ',' in our value, unable to process value. key=%s,value=%s" %
(key, value))
self.log.error("browserEnv=%s" % browserEnv)
except ValueError:
envstr += "%s%s=%s" % (delim, key, value)
delim = ","
fHandle.write("envvars=%s\n" % envstr)
fHandle.close()
self.dm.removeFile(self.remoteConfigFile)
self.dm.pushFile(fHandle.name, self.remoteConfigFile)
os.unlink(fHandle.name)
def buildBrowserEnv(self):
"""
Return an environment dictionary suitable for remote use.
This is similar to buildBrowserEnv in runtestsremote.py.
"""
browserEnv = self.environment(
xrePath=None,
debugger=None)
# remove desktop environment not used on device
if "MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA" in browserEnv:
del browserEnv["MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA"]
if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
del browserEnv["XPCOM_MEM_BLOAT_LOG"]
browserEnv["MOZ_LOG_FILE"] = os.path.join(
self.remoteMozLog,
self.mozLogName)
try:
browserEnv.update(
dict(
parseKeyValue(
self.options.environment,
context='--setenv')))
except KeyValueParseError as e:
self.log.error(str(e))
return None
return browserEnv
def runSingleTest(self, test):
"""
Run the specified test.
"""
self.log.debug("Running test %s" % test['name'])
self.mozLogName = "moz-%s.log" % test['name']
browserEnv = self.buildBrowserEnv()
self.setupRobotiumConfig(browserEnv)
self.setupRemoteProfile()
self.options.app = "am"
if self.options.autorun:
# This launches a test (using "am instrument") and instructs
# Fennec to /quit/ the browser (using Robocop:Quit) and to
# /finish/ all opened activities.
browserArgs = [
"instrument",
"-w",
"-e", "quit_and_finish", "1",
"-e", "deviceroot", self.deviceRoot,
"-e", "class",
"org.mozilla.gecko.tests.%s" % test['name'].split('/')[-1].split('.java')[0],
"org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner"]
else:
# This does not launch a test at all. It launches an activity
# that starts Fennec and then waits indefinitely, since cat
# never returns.
browserArgs = ["start",
"-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity",
"&&", "cat"]
self.dm.default_timeout = sys.maxint # Forever.
self.log.info("")
self.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
(self.options.remoteWebServer, self.options.httpPort))
self.log.info("")
result = -1
log_result = -1
try:
self.dm.recordLogcat()
timeout = self.options.timeout
if not timeout:
timeout = self.NO_OUTPUT_TIMEOUT
result = self.auto.runApp(
None, browserEnv, "am", self.localProfile, browserArgs,
timeout=timeout, symbolsPath=self.options.symbolsPath)
self.log.debug("runApp completes with status %d" % result)
if result != 0:
self.log.error("runApp() exited with code %s" % result)
if self.dm.fileExists(self.remoteLog):
self.dm.getFile(self.remoteLog, self.localLog)
self.dm.removeFile(self.remoteLog)
self.log.debug("Remote log %s retrieved to %s" %
(self.remoteLog, self.localLog))
else:
self.log.warning(
"Unable to retrieve log file (%s) from remote device" %
self.remoteLog)
log_result = self.parseLocalLog()
if result != 0 or log_result != 0:
# Display remote diagnostics; if running in mach, keep output
# terse.
if self.options.log_mach is None:
self.printDeviceInfo(printLogcat=True)
except:
self.log.error(
"Automation Error: Exception caught while running tests")
traceback.print_exc()
result = 1
self.log.debug("Test %s completes with status %d (log status %d)" %
(test['name'], int(result), int(log_result)))
return result
def runTests(self):
self.startup()
if isinstance(self.options.manifestFile, TestManifest):
mp = self.options.manifestFile
else:
mp = TestManifest(strict=False)
mp.read(self.options.robocopIni)
filters = []
if self.options.totalChunks:
filters.append(
chunk_by_slice(self.options.thisChunk, self.options.totalChunks))
robocop_tests = mp.active_tests(
exists=False, filters=filters, **mozinfo.info)
if not self.options.autorun:
# Force a single loop iteration. The iteration will start Fennec and
# the httpd server, but not actually run a test.
self.options.test_paths = [robocop_tests[0]['name']]
active_tests = []
for test in robocop_tests:
if self.options.test_paths and test['name'] not in self.options.test_paths:
continue
if 'disabled' in test:
self.log.info('TEST-INFO | skipping %s | %s' %
(test['name'], test['disabled']))
continue
active_tests.append(test)
self.log.suite_start([t['name'] for t in active_tests])
worstTestResult = None
for test in active_tests:
result = self.runSingleTest(test)
if worstTestResult is None or worstTestResult == 0:
worstTestResult = result
if worstTestResult is None:
self.log.warning(
"No tests run. Did you pass an invalid TEST_PATH?")
worstTestResult = 1
else:
print "INFO | runtests.py | Test summary: start."
logResult = self.logTestSummary()
print "INFO | runtests.py | Test summary: end."
if worstTestResult == 0:
worstTestResult = logResult
return worstTestResult
def run_test_harness(parser, options):
parser.validate(options)
if options is None:
raise ValueError(
"Invalid options specified, use --help for a list of valid options")
message_logger = MessageLogger(logger=None)
process_args = {'messageLogger': message_logger}
auto = RemoteAutomation(None, "fennec", processArgs=process_args)
auto.setDeviceManager(options.dm)
runResult = -1
robocop = RobocopTestRunner(auto, options.dm, options)
# Check that Firefox is installed
expected = options.app.split('/')[-1]
installed = options.dm.shellCheckOutput(['pm', 'list', 'packages', expected])
if expected not in installed:
robocop.log.error("%s is not installed on this device" % expected)
return 1
try:
message_logger.logger = robocop.log
message_logger.buffering = False
robocop.message_logger = message_logger
robocop.log.debug("options=%s" % vars(options))
runResult = robocop.runTests()
except KeyboardInterrupt:
robocop.log.info("runrobocop.py | Received keyboard interrupt")
runResult = -1
except:
traceback.print_exc()
robocop.log.error(
"runrobocop.py | Received unexpected exception while running tests")
runResult = 1
finally:
try:
robocop.cleanup()
except mozdevice.DMError:
# ignore device error while cleaning up
pass
message_logger.finish()
return runResult
def main(args=sys.argv[1:]):
parser = MochitestArgumentParser(app='android')
options = parser.parse_args(args)
return run_test_harness(parser, options)
if __name__ == "__main__":
sys.exit(main())