From 57827d524ca63acd4d80c229b35bf0113335b4b8 Mon Sep 17 00:00:00 2001 From: Nicolas Guichard Date: Thu, 16 Nov 2023 16:27:24 +0000 Subject: [PATCH] Bug 1490144 - Add Java and Kotlin code indexing using semanticdb compiler plugins r=nalexander,asuth,emilio,geckoview-reviewers,owlish When the Mozsearch plugin is enabled and when we compile the android target, this uses the semanticdb-javac and semanticdb-kotlinc compiler plugins to generate semanticdb files during the compilation process. In order to index all files, all files need to be compiled at least once, so this adds a `mach android compile-all` command to ensure that, otherwise some examples and AndroidTests were not necessarily compiled. Note that the AndroidTests do not have a release configuration so will not be indexed when we build in release mode. The existing searchfox mozconfigs are all set to debug so this should not be an issue. To build the android-gradle-dependencies toolchain, all dependencies must be accessible from the root build.gradle, so this also adds a flag --download-all-gradle-dependencies to ignore conditional dependency uses in gradle code. Differential Revision: https://phabricator.services.mozilla.com/D192925 --- build.gradle | 34 +++++++++++ .../android-arm-gradle-dependencies/base | 2 + mobile/android/gradle.configure | 60 +++++++++++++++++++ mobile/android/mach_commands.py | 12 ++++ toolkit/mozapps/installer/package-name.mk | 1 + toolkit/mozapps/installer/packager.mk | 8 +++ toolkit/mozapps/installer/upload-files.mk | 3 + 7 files changed, 120 insertions(+) diff --git a/build.gradle b/build.gradle index 53f8a7a17be6..5e222ceaadf7 100644 --- a/build.gradle +++ b/build.gradle @@ -70,6 +70,40 @@ allprojects { } } + // Use the semanticdb-javac and semanticdb-kotlinc plugins to generate semanticdb files for Searchfox + if (mozconfig.substs.ENABLE_MOZSEARCH_PLUGIN || mozconfig.substs.DOWNLOAD_ALL_GRADLE_DEPENDENCIES) { + def targetRoot = new File(topobjdir, "mozsearch_java_index") + def semanticdbJavacVersion = "com.sourcegraph:semanticdb-javac:0.9.6" + def semanticdbKotlincVersion = "com.sourcegraph:semanticdb-kotlinc:0.3.2" + + afterEvaluate { + def addDependencyToConfigurationIfExists = { configurationName, dependency -> + def configuration = configurations.findByName(configurationName) + if (configuration != null) { + dependencies.add(configurationName, dependency) + } + } + + addDependencyToConfigurationIfExists("compileOnly", semanticdbJavacVersion) + addDependencyToConfigurationIfExists("testCompileOnly", semanticdbJavacVersion) + addDependencyToConfigurationIfExists("androidTestCompileOnly", semanticdbJavacVersion) + addDependencyToConfigurationIfExists("kotlinCompilerPluginClasspath", semanticdbKotlincVersion) + } + + tasks.withType(JavaCompile) { + options.compilerArgs += [ + "-Xplugin:semanticdb -sourceroot:${topsrcdir} -targetroot:${targetRoot}", + ] + } + + tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile) { + compilerOptions.freeCompilerArgs.addAll([ + "-P", "plugin:semanticdb-kotlinc:sourceroot=${topsrcdir}", + "-P", "plugin:semanticdb-kotlinc:targetroot=${targetRoot}", + ]) + } + } + task downloadDependencies() { description 'Download all dependencies to the Gradle cache' doLast { diff --git a/mobile/android/config/mozconfigs/android-arm-gradle-dependencies/base b/mobile/android/config/mozconfigs/android-arm-gradle-dependencies/base index d246cf24d722..7384a99e0d33 100644 --- a/mobile/android/config/mozconfigs/android-arm-gradle-dependencies/base +++ b/mobile/android/config/mozconfigs/android-arm-gradle-dependencies/base @@ -15,6 +15,8 @@ ac_add_options --with-gradle export GRADLE_MAVEN_REPOSITORIES="http://localhost:8081/nexus/content/repositories/mozilla/","http://localhost:8081/nexus/content/repositories/google/","http://localhost:8081/nexus/content/repositories/central/","http://localhost:8081/nexus/content/repositories/gradle-plugins/" # Nexus runs on HTTP ac_add_options --allow-insecure-gradle-repositories +# Some dependencies may be conditionally-loaded (eg. semanticdb compiler plugins) +ac_add_options --download-all-gradle-dependencies # From here on, just like ../android-arm/nightly. diff --git a/mobile/android/gradle.configure b/mobile/android/gradle.configure index 557f16c4361e..d13290c1d9a0 100644 --- a/mobile/android/gradle.configure +++ b/mobile/android/gradle.configure @@ -499,6 +499,55 @@ set_config( ) +@depends(gradle_android_build_config) +def gradle_android_compile_all_tasks(build_config): + """Gradle tasks run by |mach android compile-all|.""" + + def capitalize(s): + # str.capitalize lower cases trailing letters. + if s: + return s[0].upper() + s[1:] + else: + return s + + buildType = capitalize(build_config.geckoview.variant.buildType) + + tasks = [ + f"compileJava", + f"compileTestJava", + f"compile{buildType}Sources", + f"compile{buildType}UnitTestSources", + f"geckoview:compile{build_config.geckoview.variant.name}Sources", + f"geckoview:compile{build_config.geckoview.variant.name}UnitTestSources", + f"test_runner:compile{build_config.geckoview.variant.name}Sources", + f"test_runner:compile{build_config.geckoview.variant.name}UnitTestSources", + f"messaging_example:compile{build_config.geckoview.variant.name}Sources", + f"messaging_example:compile{build_config.geckoview.variant.name}UnitTestSources", + f"port_messaging_example:compile{build_config.geckoview.variant.name}Sources", + f"port_messaging_example:compile{build_config.geckoview.variant.name}UnitTestSources", + f"geckoview_example:compile{build_config.geckoview_example.variant.name}Sources", + f"geckoview_example:compile{build_config.geckoview_example.variant.name}UnitTestSources", + ] + + if buildType == "Debug": + tasks += [ + f"compile{buildType}AndroidTestSources", + f"geckoview:compile{build_config.geckoview.variant.name}AndroidTestSources", + f"test_runner:compile{build_config.geckoview.variant.name}AndroidTestSources", + f"messaging_example:compile{build_config.geckoview.variant.name}AndroidTestSources", + f"port_messaging_example:compile{build_config.geckoview.variant.name}AndroidTestSources", + f"geckoview_example:compile{build_config.geckoview_example.variant.name}AndroidTestSources", + ] + + return tasks + + +set_config( + "GRADLE_ANDROID_COMPILE_ALL_TASKS", + gradle_android_compile_all_tasks, +) + + @depends(gradle_android_build_config) def gradle_android_install_geckoview_test_runner_tasks(build_config): """Gradle tasks run by |mach android install-geckoview-test_runner|.""" @@ -604,6 +653,17 @@ set_config( when="--allow-insecure-gradle-repositories", ) +option( + "--download-all-gradle-dependencies", + help="Download all dependencies, even those that are conditionally used.", +) + +set_config( + "DOWNLOAD_ALL_GRADLE_DEPENDENCIES", + True, + when="--download-all-gradle-dependencies", +) + @depends("GRADLE_MAVEN_REPOSITORIES") @imports(_from="os.path", _import="isdir") diff --git a/mobile/android/mach_commands.py b/mobile/android/mach_commands.py index 1285095bd4bf..8a145ed95d07 100644 --- a/mobile/android/mach_commands.py +++ b/mobile/android/mach_commands.py @@ -210,6 +210,18 @@ def android_build_geckoview_example(command_context, args): return 0 +@SubCommand("android", "compile-all", """Build all source files""") +@CommandArgument("args", nargs=argparse.REMAINDER) +def android_compile_all(command_context, args): + gradle( + command_context, + command_context.substs["GRADLE_ANDROID_COMPILE_ALL_TASKS"] + args, + verbose=True, + ) + + return 0 + + def install_app_bundle(command_context, bundle): from mozdevice import ADBDeviceFactory diff --git a/toolkit/mozapps/installer/package-name.mk b/toolkit/mozapps/installer/package-name.mk index cc31b66ed7a8..f0ac4d6d2ea0 100644 --- a/toolkit/mozapps/installer/package-name.mk +++ b/toolkit/mozapps/installer/package-name.mk @@ -92,6 +92,7 @@ CODE_COVERAGE_ARCHIVE_BASENAME = $(PKG_BASENAME).code-coverage-gcno MOZSEARCH_ARCHIVE_BASENAME = $(PKG_BASENAME).mozsearch-index MOZSEARCH_INCLUDEMAP_BASENAME = $(PKG_BASENAME).mozsearch-distinclude MOZSEARCH_SCIP_INDEX_BASENAME = $(PKG_BASENAME).mozsearch-scip-index +MOZSEARCH_JAVA_INDEX_BASENAME = $(PKG_BASENAME).mozsearch-java-index # Mozharness naming MOZHARNESS_PACKAGE = mozharness.zip diff --git a/toolkit/mozapps/installer/packager.mk b/toolkit/mozapps/installer/packager.mk index 03393b885b4f..aeba614e40fd 100644 --- a/toolkit/mozapps/installer/packager.mk +++ b/toolkit/mozapps/installer/packager.mk @@ -91,6 +91,14 @@ ifdef ENABLE_MOZSEARCH_PLUGIN zip -r5D '$(ABS_DIST)/$(PKG_PATH)$(MOZSEARCH_SCIP_INDEX_BASENAME).zip' \ index.scip rm $(topsrcdir)/.cargo/config +ifeq ($(MOZ_BUILD_APP),mobile/android) + @echo 'Generating mozsearch java/kotlin semanticdb tarball...' + $(RM) $(MOZSEARCH_JAVA_INDEX_BASENAME).zip + cd $(topsrcdir)/ && \ + $(PYTHON3) $(topsrcdir)/mach android compile-all && \ + cd $(topobjdir)/mozsearch_java_index && \ + zip -r5D '$(ABS_DIST)/$(PKG_PATH)$(MOZSEARCH_JAVA_INDEX_BASENAME).zip' . +endif # MOZ_BUILD_APP == mobile/android endif ifeq (Darwin, $(OS_ARCH)) ifneq (,$(MOZ_ASAN)$(LIBFUZZER)$(MOZ_UBSAN)) diff --git a/toolkit/mozapps/installer/upload-files.mk b/toolkit/mozapps/installer/upload-files.mk index 80ff68626aa0..62499172de2e 100644 --- a/toolkit/mozapps/installer/upload-files.mk +++ b/toolkit/mozapps/installer/upload-files.mk @@ -383,6 +383,9 @@ ifdef ENABLE_MOZSEARCH_PLUGIN UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZSEARCH_ARCHIVE_BASENAME).zip) UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZSEARCH_SCIP_INDEX_BASENAME).zip) UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZSEARCH_INCLUDEMAP_BASENAME).map) +ifeq ($(MOZ_BUILD_APP),mobile/android) + UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZSEARCH_JAVA_INDEX_BASENAME).zip) +endif endif ifdef MOZ_STUB_INSTALLER