android/gradle/with_gecko_binaries.gradle

/* -*- Mode: Groovy; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// The JNI wrapper generation tasks depend on the JAR creation task of the :annotations project.
evaluationDependsOn(':annotations')

import groovy.util.FileNameFinder
import groovy.transform.Memoized

import java.nio.file.Path
import java.nio.file.Paths
import java.security.MessageDigest

// To find the Android NDK directory, there are a few wrinkles.  In a compiled
// build, we can use our own `ANDROID_NDK` configure option.  But in an
// artifact build, that isn't defined, so we fall back to
// `android.ndkDirectory` -- but that's defined in `local.properties`, which
// may not define it.  In that case, fall back to crawling the filesystem.
def getNDKDirectory() {
    if (project.mozconfig.substs.ANDROID_NDK) {
        return project.mozconfig.substs.ANDROID_NDK
    }
    try {
        if (project.android.ndkDirectory) {
            return project.android.ndkDirectory
        }
    } catch (InvalidUserDataException ex) {
        // The NDK is not installed, that's ok.
    }
    def mozbuild = System.getenv('MOZBUILD_STATE_PATH') ?: "${System.getProperty('user.home')}/.mozbuild"
    def files = new FileNameFinder().getFileNames(mozbuild, "android-ndk-*/source.properties")
    if (files) {
        // It would be nice to sort these by version, but that's too much effort right now.
        return project.file(files.first()).parentFile.toString()
    }
    return null
}

def hasCompileArtifacts() {
    return project.mozconfig.substs.COMPILE_ENVIRONMENT
        || project.mozconfig.substs.MOZ_ARTIFACT_BUILDS
}

// Get the LLVM bin folder, either from the currently used toolchain or, if
// this is an artifact build, from the Android NDK.
@Memoized
def getLlvmBinFolder() {
    // If we have a clang path, just use that
    if (project.mozconfig.substs.MOZ_CLANG_PATH) {
        return project.file(project.mozconfig.substs.MOZ_CLANG_PATH)
            .parentFile
    }

    def ndkDirectory = getNDKDirectory()
    if (!ndkDirectory) {
        // Give up
        return null;
    }

    // The `**` in the pattern depends on the host architecture.  We could compute them or bake them
    // in, but this is easy enough.  `FileNameFinder` won't return a directory, so we find a file
    // and return its parent directory.
    return project
        .file(new FileNameFinder()
                  .getFileNames(ndkDirectory, "toolchains/llvm/prebuilt/**/bin/llvm-*")
                  .first())
        .parentFile
}

ext.getLlvmBinFolder = {
    // This is the easiest way to expose the memoized function to consumers.
    getLlvmBinFolder()
}

// Bug 1657190: This task works around a bug in the Android-Gradle plugin.  When using SNAPSHOT
// builds (and possibly in other situations) the JNI library invalidation process isn't correct.  If
// there are two JNI libs `a.so` and `b.so` and a new snapshot updates only `a.so`, then the
// resulting AAR will include both JNI libs but the consuming project's resulting APK will include
// only the modified `a.so`.  For GeckoView, it's usually `libxul.so` that is updated.  For a
// consumer (like Fenix), this leads to a hard startup crash because `libmozglue.so` will be missing
// when it is read (first, before `libxul.so`)
//
// For !MOZILLA_OFFICIAL builds, we work around this issue to ensure that when *any* library is
// updated then *every* library is updated.  We use the ELF build ID baked into each library to
// determine whether any library is updated.  We digest (SHA256, for now) all of the ELF build IDs
// and use this as our own Mozilla-specific "library generation ID".  We then add our own
// Mozilla-specific ELF section to every library so that they will all be invalidated by the
// Android-Gradle plugin of a consuming project.
class SyncLibsAndUpdateGenerationID extends DefaultTask {
    @InputDirectory
    File source

    @InputFile
    File extraSource

    @OutputDirectory
    File destinationDir

    @InputDirectory
    File llvmBin = project.ext.getLlvmBinFolder()

    // Sibling to `.note.gnu.build-id`.
    @Input
    String newElfSection = ".note.mozilla.build-id"

    @TaskAction
    void execute() {
        Path root = Paths.get(source.toString())

        def libs = project.fileTree(source.toString()).files.collect {
            Path lib = Paths.get(it.toString())
            root.relativize(lib).toString()
        }

        def generationId = new ByteArrayOutputStream().withStream { os ->
            // Start with the hash of any extra source.
            def digest = MessageDigest.getInstance("SHA-256")
            extraSource.eachByte(64 * 1024) { byte[] buf, int bytesRead ->
                digest.update(buf, 0, bytesRead);
            }
            def extraSourceHex = new BigInteger(1, digest.digest()).toString(16).padLeft(64, '0')
            os.write("${extraSource.toString()}  ${extraSourceHex}\n".getBytes("utf-8"));

            // Follow with all the ordered build ID section dumps.
            libs.each { lib ->
                // This should never fail.  If it does, there's a real problem, so an exception is
                // reasonable.
                project.exec {
                    commandLine "${llvmBin}/llvm-readobj"
                    args '--hex-dump=.note.gnu.build-id'
                    args "${source}/${lib}"
                    standardOutput = os
                }
            }

            def allBuildIDs = os.toString()

            // For detailed debugging.
            new File("${destinationDir}/${newElfSection}-details").setText(allBuildIDs, 'utf-8')

            allBuildIDs.sha256()
        }

        logger.info("Mozilla-specific library generation ID is ${generationId} (see ${destinationDir}/${newElfSection}-details)")

        def generationIdIsStale = libs.any { lib ->
            new ByteArrayOutputStream().withStream { os ->
                // This can fail, but there's little value letting stderr go to the console in
                // general, since it's just noise after a clobber build.
                def execResult = project.exec {
                    ignoreExitValue true
                    commandLine "${llvmBin}/llvm-readobj"
                    args "--hex-dump=${newElfSection}"
                    args "${destinationDir}/${lib}"
                    standardOutput = os
                    errorOutput = new ByteArrayOutputStream()
                }

                if (execResult.exitValue != 0) {
                    logger.info("Mozilla-specific library generation ID is missing: ${lib}")
                } else if (!os.toString().contains(generationId)) {
                    logger.info("Mozilla-specific library generation ID is stale: ${lib}")
                } else {
                    logger.debug("Mozilla-specific library generation ID is fresh: ${lib}")
                }
                execResult.exitValue != 0 || !os.toString().contains(generationId)
            }
        }

        if (generationIdIsStale) {
            project.mkdir destinationDir
            new File("${destinationDir}/${newElfSection}").setText(generationId, 'utf-8')

            libs.each { lib ->
                project.mkdir project.file("${destinationDir}/${lib}").parent

                new ByteArrayOutputStream().withStream { os ->
                    // This should never fail: if it does, there's a real problem (again).
                    project.exec {
                        commandLine "${llvmBin}/llvm-objcopy"
                        args "--add-section=${newElfSection}=${destinationDir}/${newElfSection}"
                        args "${source}/${lib}"
                        args "${destinationDir}/${lib}"
                        standardOutput = os
                    }
                }
            }
        } else {
            logger.info("Mozilla-specific library generation ID is fresh!")
        }
    }
}

ext.configureVariantWithGeckoBinaries = { variant ->
    def omnijarDir = "${topobjdir}/dist/geckoview"
    def distDir = "${topobjdir}/dist/geckoview"

    def syncOmnijarFromDistDir
    if (hasCompileArtifacts()) {
        syncOmnijarFromDistDir = task("syncOmnijarFromDistDirFor${variant.name.capitalize()}", type: Sync) {
            onlyIf {
                if (source.empty) {
                    throw new StopExecutionException("Required omnijar not found in ${omnijarDir}/{omni.ja,assets/omni.ja}.  Have you built and packaged?")
                }
                return true
            }

            into("${project.buildDir}/moz.build/src/${variant.name}/omnijar")
            from("${omnijarDir}/omni.ja",
                 "${omnijarDir}/assets/omni.ja") {
                // Throw an exception if we find multiple, potentially conflicting omni.ja files.
                duplicatesStrategy 'fail'
            }
        }
    }

    // For !MOZILLA_OFFICIAL builds, work around an Android-Gradle plugin bug that causes startup
    // crashes with local substitution.  But -- we want to allow artifact builds that don't have the
    // NDK installed.  See class comment above.
    def shouldUpdateGenerationID = {
        if (mozconfig.substs.MOZILLA_OFFICIAL) {
            return false
        } else if (mozconfig.substs.MOZ_ANDROID_FAT_AAR_ARCHITECTURES) {
            return false
        } else if (ext.getLlvmBinFolder() == null) {
            logger.warn("Could not determine LLVM bin directory.")
            logger.warn("Set `ndk.dir` in `${project.topsrcdir}/local.properties` to avoid startup crashes when using `substitute-local-geckoview.gradle`.")
            logger.warn("See https://bugzilla.mozilla.org/show_bug.cgi?id=1657190.")
            return false
        }
        return true
    }()

    def syncLibsFromDistDir = { if (shouldUpdateGenerationID) {
        def jarTask = tasks["bundleLibRuntimeToJar${variant.name.capitalize()}"]
        def bundleJar = jarTask.outputs.files.find({ it.name == 'classes.jar' })

        task("syncLibsAndUpdateGenerationIDFromDistDirFor${variant.name.capitalize()}", type: SyncLibsAndUpdateGenerationID) {
            source file("${distDir}/lib")
            destinationDir file("${project.buildDir}/moz.build/src/${variant.name}/jniLibs")
            // Include the hash of classes.jar as well, so that JVM-only changes don't leave every
            // JNI library unchanged and therefore invalidate all of the JNI libraries in a consumer
            // doing local substitution.
            extraSource bundleJar
            dependsOn jarTask
        }
    } else {
        task("syncLibsFromDistDirFor${variant.name.capitalize()}", type: Sync) {
            into("${project.buildDir}/moz.build/src/${variant.name}/jniLibs")
            from("${distDir}/lib")
        }
    } }()

    syncLibsFromDistDir.onlyIf { task ->
        if (!hasCompileArtifacts()) {
            // We won't have JNI libraries if we're not compiling and we're not downloading
            // artifacts.  Such a configuration is used for running lints, generating docs, etc.
            return true
        }
        if (files(task.source).empty) {
            throw new StopExecutionException("Required JNI libraries not found in ${task.source}.  Have you built and packaged?")
        }
        return true
    }

    def syncAssetsFromDistDir = task("syncAssetsFromDistDirFor${variant.name.capitalize()}", type: Sync) {
        into("${project.buildDir}/moz.build/src/${variant.name}/assets")
        from("${distDir}/assets") {
            exclude 'omni.ja'
        }
    }

    if (hasCompileArtifacts()) {
        // Local (read, not 'official') builds want to reflect developer changes to
        // the omnijar sources, and (when compiling) to reflect developer changes to
        // the native binaries.  To do this, the Gradle build calls out to the
        // moz.build system, which can be re-entrant.  Official builds are driven by
        // the moz.build system and should never be re-entrant in this way.
        if (!mozconfig.substs.MOZILLA_OFFICIAL) {
            syncOmnijarFromDistDir.dependsOn rootProject.machStagePackage
            syncLibsFromDistDir.dependsOn rootProject.machStagePackage
            syncAssetsFromDistDir.dependsOn rootProject.machStagePackage
        }

        def assetGenTask = tasks.findByName("generate${variant.name.capitalize()}Assets")
        def jniLibFoldersTask = tasks.findByName("merge${variant.name.capitalize()}JniLibFolders")
        if ((variant.productFlavors*.name).contains('withGeckoBinaries')) {
            assetGenTask.dependsOn syncOmnijarFromDistDir
            assetGenTask.dependsOn syncAssetsFromDistDir
            jniLibFoldersTask.dependsOn syncLibsFromDistDir

            android.sourceSets."${variant.name}".assets.srcDir syncOmnijarFromDistDir.destinationDir
            android.sourceSets."${variant.name}".assets.srcDir syncAssetsFromDistDir.destinationDir

            if (!mozconfig.substs.MOZ_ANDROID_FAT_AAR_ARCHITECTURES) {
                android.sourceSets."${variant.name}".jniLibs.srcDir syncLibsFromDistDir.destinationDir
            } else {
                android.sourceSets."${variant.name}".jniLibs.srcDir "${topobjdir}/dist/fat-aar/output/jni"
            }
        }
    }
}

ext.configureLibraryVariantWithJNIWrappers = { variant, module ->
    // BundleLibRuntime prepares the library for further processing to be
    // incorporated in an app. We use this version to create the JNI wrappers.
    def jarTask = tasks["bundleLibRuntimeToJar${variant.name.capitalize()}"]
    def bundleJar = jarTask.outputs.files.find({ it.name == 'classes.jar' })

    def annotationProcessorsJarTask = project(':annotations').jar

    def wrapperTask
    if (System.env.IS_LANGUAGE_REPACK == '1') {
        // Single-locale l10n repacks set `IS_LANGUAGE_REPACK=1` and don't
        // really have a build environment.
        wrapperTask = task("generateJNIWrappersFor${module}${variant.name.capitalize()}")
    } else {
        wrapperTask = task("generateJNIWrappersFor${module}${variant.name.capitalize()}", type: JavaExec) {
            classpath annotationProcessorsJarTask.archivePath

            // Configure the classpath at evaluation-time, not at
            // configuration-time: see above comment.
            doFirst {
                classpath variant.javaCompileProvider.get().classpath
            }

            mainClass = 'org.mozilla.gecko.annotationProcessors.AnnotationProcessor'
            args module
            args bundleJar

            workingDir "${topobjdir}/widget/android"

            inputs.file(bundleJar)
            inputs.file(annotationProcessorsJarTask.archivePath)
            inputs.property("module", module)

            outputs.file("${topobjdir}/widget/android/GeneratedJNINatives.h")
            outputs.file("${topobjdir}/widget/android/GeneratedJNIWrappers.cpp")
            outputs.file("${topobjdir}/widget/android/GeneratedJNIWrappers.h")

            dependsOn jarTask
            dependsOn annotationProcessorsJarTask
        }
    }

    if (module == 'Generated') {
        tasks["bundle${variant.name.capitalize()}Aar"].dependsOn wrapperTask
    } else {
        tasks["assemble${variant.name.capitalize()}"].dependsOn wrapperTask
    }
}