Bug 1490144 - Add Java and Kotlin code indexing using semanticdb compiler plugins r=nalexander,asuth,emilio,geckoview-reviewers,owlish

When the Mozsearch plugin is enabled and when we compile the android
target, this uses the semanticdb-javac and semanticdb-kotlinc compiler
plugins to generate semanticdb files during the compilation process.

In order to index all files, all files need to be compiled at least
once, so this adds a `mach android compile-all` command to ensure that,
otherwise some examples and AndroidTests were not necessarily compiled.
Note that the AndroidTests do not have a release configuration so will
not be indexed when we build in release mode. The existing searchfox
mozconfigs are all set to debug so this should not be an issue.

To build the android-gradle-dependencies toolchain, all dependencies
must be accessible from the root build.gradle, so this also adds a flag
--download-all-gradle-dependencies to ignore conditional dependency
uses in gradle code.

Differential Revision: https://phabricator.services.mozilla.com/D192925

build.gradle

@@ -70,6 +70,40 @@ allprojects {
         }
     }
 
+    // Use the semanticdb-javac and semanticdb-kotlinc plugins to generate semanticdb files for Searchfox
+    if (mozconfig.substs.ENABLE_MOZSEARCH_PLUGIN || mozconfig.substs.DOWNLOAD_ALL_GRADLE_DEPENDENCIES) {
+        def targetRoot = new File(topobjdir, "mozsearch_java_index")
+        def semanticdbJavacVersion = "com.sourcegraph:semanticdb-javac:0.9.6"
+        def semanticdbKotlincVersion = "com.sourcegraph:semanticdb-kotlinc:0.3.2"
+
+        afterEvaluate {
+            def addDependencyToConfigurationIfExists = { configurationName, dependency ->
+                def configuration = configurations.findByName(configurationName)
+                if (configuration != null) {
+                    dependencies.add(configurationName, dependency)
+                }
+            }
+
+            addDependencyToConfigurationIfExists("compileOnly", semanticdbJavacVersion)
+            addDependencyToConfigurationIfExists("testCompileOnly", semanticdbJavacVersion)
+            addDependencyToConfigurationIfExists("androidTestCompileOnly", semanticdbJavacVersion)
+            addDependencyToConfigurationIfExists("kotlinCompilerPluginClasspath", semanticdbKotlincVersion)
+        }
+
+        tasks.withType(JavaCompile) {
+            options.compilerArgs += [
+                "-Xplugin:semanticdb -sourceroot:${topsrcdir} -targetroot:${targetRoot}",
+            ]
+        }
+
+        tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile) {
+            compilerOptions.freeCompilerArgs.addAll([
+                "-P", "plugin:semanticdb-kotlinc:sourceroot=${topsrcdir}",
+                "-P", "plugin:semanticdb-kotlinc:targetroot=${targetRoot}",
+            ])
+        }
+    }
+
     task downloadDependencies() {
         description 'Download all dependencies to the Gradle cache'
         doLast {

mobile/android/config/mozconfigs/android-arm-gradle-dependencies/base

@@ -15,6 +15,8 @@ ac_add_options --with-gradle
 export GRADLE_MAVEN_REPOSITORIES="http://localhost:8081/nexus/content/repositories/mozilla/","http://localhost:8081/nexus/content/repositories/google/","http://localhost:8081/nexus/content/repositories/central/","http://localhost:8081/nexus/content/repositories/gradle-plugins/"
 # Nexus runs on HTTP
 ac_add_options --allow-insecure-gradle-repositories
+# Some dependencies may be conditionally-loaded (eg. semanticdb compiler plugins)
+ac_add_options --download-all-gradle-dependencies
 
 # From here on, just like ../android-arm/nightly.
 

mobile/android/gradle.configure

@@ -499,6 +499,55 @@ set_config(
 )
 
 
+@depends(gradle_android_build_config)
+def gradle_android_compile_all_tasks(build_config):
+    """Gradle tasks run by |mach android compile-all|."""
+
+    def capitalize(s):
+        # str.capitalize lower cases trailing letters.
+        if s:
+            return s[0].upper() + s[1:]
+        else:
+            return s
+
+    buildType = capitalize(build_config.geckoview.variant.buildType)
+
+    tasks = [
+        f"compileJava",
+        f"compileTestJava",
+        f"compile{buildType}Sources",
+        f"compile{buildType}UnitTestSources",
+        f"geckoview:compile{build_config.geckoview.variant.name}Sources",
+        f"geckoview:compile{build_config.geckoview.variant.name}UnitTestSources",
+        f"test_runner:compile{build_config.geckoview.variant.name}Sources",
+        f"test_runner:compile{build_config.geckoview.variant.name}UnitTestSources",
+        f"messaging_example:compile{build_config.geckoview.variant.name}Sources",
+        f"messaging_example:compile{build_config.geckoview.variant.name}UnitTestSources",
+        f"port_messaging_example:compile{build_config.geckoview.variant.name}Sources",
+        f"port_messaging_example:compile{build_config.geckoview.variant.name}UnitTestSources",
+        f"geckoview_example:compile{build_config.geckoview_example.variant.name}Sources",
+        f"geckoview_example:compile{build_config.geckoview_example.variant.name}UnitTestSources",
+    ]
+
+    if buildType == "Debug":
+        tasks += [
+            f"compile{buildType}AndroidTestSources",
+            f"geckoview:compile{build_config.geckoview.variant.name}AndroidTestSources",
+            f"test_runner:compile{build_config.geckoview.variant.name}AndroidTestSources",
+            f"messaging_example:compile{build_config.geckoview.variant.name}AndroidTestSources",
+            f"port_messaging_example:compile{build_config.geckoview.variant.name}AndroidTestSources",
+            f"geckoview_example:compile{build_config.geckoview_example.variant.name}AndroidTestSources",
+        ]
+
+    return tasks
+
+
+set_config(
+    "GRADLE_ANDROID_COMPILE_ALL_TASKS",
+    gradle_android_compile_all_tasks,
+)
+
+
 @depends(gradle_android_build_config)
 def gradle_android_install_geckoview_test_runner_tasks(build_config):
     """Gradle tasks run by |mach android install-geckoview-test_runner|."""
@@ -604,6 +653,17 @@ set_config(
     when="--allow-insecure-gradle-repositories",
 )
 
+option(
+    "--download-all-gradle-dependencies",
+    help="Download all dependencies, even those that are conditionally used.",
+)
+
+set_config(
+    "DOWNLOAD_ALL_GRADLE_DEPENDENCIES",
+    True,
+    when="--download-all-gradle-dependencies",
+)
+
 
 @depends("GRADLE_MAVEN_REPOSITORIES")
 @imports(_from="os.path", _import="isdir")

mobile/android/mach_commands.py

@@ -210,6 +210,18 @@ def android_build_geckoview_example(command_context, args):
     return 0
 
 
+@SubCommand("android", "compile-all", """Build all source files""")
+@CommandArgument("args", nargs=argparse.REMAINDER)
+def android_compile_all(command_context, args):
+    gradle(
+        command_context,
+        command_context.substs["GRADLE_ANDROID_COMPILE_ALL_TASKS"] + args,
+        verbose=True,
+    )
+
+    return 0
+
+
 def install_app_bundle(command_context, bundle):
     from mozdevice import ADBDeviceFactory
 

toolkit/mozapps/installer/package-name.mk

@@ -92,6 +92,7 @@ CODE_COVERAGE_ARCHIVE_BASENAME = $(PKG_BASENAME).code-coverage-gcno
 MOZSEARCH_ARCHIVE_BASENAME = $(PKG_BASENAME).mozsearch-index
 MOZSEARCH_INCLUDEMAP_BASENAME = $(PKG_BASENAME).mozsearch-distinclude
 MOZSEARCH_SCIP_INDEX_BASENAME = $(PKG_BASENAME).mozsearch-scip-index
+MOZSEARCH_JAVA_INDEX_BASENAME = $(PKG_BASENAME).mozsearch-java-index
 
 # Mozharness naming
 MOZHARNESS_PACKAGE = mozharness.zip

toolkit/mozapps/installer/packager.mk

@@ -91,6 +91,14 @@ ifdef ENABLE_MOZSEARCH_PLUGIN
           zip -r5D '$(ABS_DIST)/$(PKG_PATH)$(MOZSEARCH_SCIP_INDEX_BASENAME).zip' \
           index.scip
 	rm $(topsrcdir)/.cargo/config
+ifeq ($(MOZ_BUILD_APP),mobile/android)
+	@echo 'Generating mozsearch java/kotlin semanticdb tarball...'
+	$(RM) $(MOZSEARCH_JAVA_INDEX_BASENAME).zip
+	cd $(topsrcdir)/ && \
+          $(PYTHON3) $(topsrcdir)/mach android compile-all && \
+          cd $(topobjdir)/mozsearch_java_index && \
+          zip -r5D '$(ABS_DIST)/$(PKG_PATH)$(MOZSEARCH_JAVA_INDEX_BASENAME).zip' .
+endif # MOZ_BUILD_APP == mobile/android
 endif
 ifeq (Darwin, $(OS_ARCH))
 ifneq (,$(MOZ_ASAN)$(LIBFUZZER)$(MOZ_UBSAN))

toolkit/mozapps/installer/upload-files.mk

@@ -383,6 +383,9 @@ ifdef ENABLE_MOZSEARCH_PLUGIN
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZSEARCH_ARCHIVE_BASENAME).zip)
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZSEARCH_SCIP_INDEX_BASENAME).zip)
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZSEARCH_INCLUDEMAP_BASENAME).map)
+ifeq ($(MOZ_BUILD_APP),mobile/android)
+  UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZSEARCH_JAVA_INDEX_BASENAME).zip)
+endif
 endif
 
 ifdef MOZ_STUB_INSTALLER