オンデバイス インテリジェンス: Android ML Kit API を使用してパフォーマンスと品質を最適化

val translator = SpeechTranslation.getClient(
    SpeechTranslatorOptions.Builder(
        Locale.JAPANESE, Locale.US
    ).build()
)

val translator = SpeechTranslation.getClient(
    SpeechTranslatorOptions.Builder(
        Locale.JAPANESE, Locale.US
    ).build()
)

val audioFlow = flow {
    emit(TranslationAudio(Locale.JAPANESE, courierVoicemailBytes))
}

val request = SpeechTranslatorRequest.Builder(audioFlow).build()
translator.startTranslation(request).collect { response ->
    if (response is SpeechTranslatorResponse.Text) {
        translatedCourierClaim = response.translatedText
    }
}

val recognizer = SpeechRecognition.getClient(
    speechRecognizerOptions {
        locale = Locale.US
        preferredMode = SpeechRecognizerOptions.Mode.MODE_ADVANCED
    }
)

val recognizer = SpeechRecognition.getClient(
    speechRecognizerOptions {
        locale = Locale.US
        preferredMode = SpeechRecognizerOptions.Mode.MODE_ADVANCED
    }
)

val request = speechRecognizerRequest {
    audioSource = AudioSource.fromMic()
    recognitionOptions { enableWordTiming = true } // optional
}

recognizer.startRecognition(request).collect { response ->
    when (response) {
        is SpeechRecognizerResponse.PartialTextResponse ->
            liveCaption = response.text
        is SpeechRecognizerResponse.FinalTextResponse ->
            planAndCallVaultTool(translatedCourierClaim, response.text)
    }
}

retrieveOrderAndDeliveryPhotos(
  searchQuery = "camera lens customs",
  claimedFeeUsd = 45,
  comparePackagePhotos = true
)

// Keep SystemInstruction static across Turn 1 & Turn 2:
val securitySystemInstruction = SystemInstruction(
    """
    You are an on-device parcel & phishing security auditor.
    1. Cross-check packaging, carrier labels and tracking across all images.
    2. Verify customs fee claims against local DDP (Delivered Duty Paid) order records.
    3. Flag any payment demand when the order record shows duties prepaid.
    """.trimIndent()
)

// Pull in the Speech Translation (AST) & Speech Recognition (ASR) outputs:
val turn1Prompt = """
    Translated voicemail: $translatedCourierClaim
    Incoming text message: $callerText
    User's spoken question: $userQuestion
    """.trimIndent()

val turn1Request = generateContentRequest(
    securitySystemInstruction,
    TextPart(turn1Prompt)
) { /* enableThinking & setTools */ }

// Registered in AndroidManifest.xml
@AppFunctionServiceEntryPoint("ScamInvestigationService", "scam_tools")
abstract class ScamInvestigationTools : AppFunctionService() {

}

// Registered in AndroidManifest.xml
@AppFunctionServiceEntryPoint("ScamInvestigationService", "scam_tools")
abstract class ScamInvestigationTools : AppFunctionService() {
    /**
     * Searches local orders and order images in AppSearch.
     * @param searchQuery Semantic query to match the order
     * @param claimedFeeUsd Customs fee claimed in USD
     * @param comparePackagePhotos Whether to fetch order images to compare
     */
    @AppFunctionDeclaration(isDescribedByKDoc = true)
    suspend fun retrieveOrderAndDeliveryPhotos(
        searchQuery: String,
        claimedFeeUsd: Int,
        comparePackagePhotos: Boolean
    ): VaultMatch = withContext(Dispatchers.IO) { executeVaultSearch(searchQuery) }
}

val vaultTools = AppFunctionTools.getClient().getAppFunctions()
val turn1Request = generateContentRequest(securitySystemInstruction, TextPart(turn1Prompt)) {

    enableThinking = true
    setTools(vaultTools)

}
val call = generativeModel.generateContent(turn1Request)
    .candidates.first().functionCalls.first()

val embedModel = EmbeddingGeneration.getEmbeddingModels().first()
val docVector = embedModel.generateEmbedding(
    EmbeddingRequest(
        Content(listOf(
            ImagePart(orderEmailLensBmp),   // the lens photo in the email
            TextPart(orderEmailBody)        // the order details
        ))
    )
).embedding.vector

val embedModel = EmbeddingGeneration.getEmbeddingModels().first()
val docVector = embedModel.generateEmbedding(
    EmbeddingRequest(
        Content(listOf(
            ImagePart(orderEmailLensBmp),   // the lens photo in the email
            TextPart(orderEmailBody)        // the order details
        ))
    )
).embedding.vector

val doc = GenericDocument.Builder<GenericDocument.Builder<*>>(
    "orders", "order_jp_904", "OrderRecord"
)
    .setPropertyString("summary", "Shipped DDP, duties prepaid ($0 owed). In transit.")
    .setPropertyString("photoUris", "order_email_lens.png")
    .setPropertyEmbedding(
        "embedding",
        EmbeddingVector(docVector, embedModel.getModelSignature())
    ).build()

suspend fun executeVaultSearch(searchQuery: String): VaultMatch {
    val qVec = embedModel.generateEmbedding(
        EmbeddingRequest(
            content = Content(listOf(TextPart(searchQuery)))
        )
    ).embedding.vector

}

suspend fun executeVaultSearch(searchQuery: String): VaultMatch {
    val qVec = embedModel.generateEmbedding(
        EmbeddingRequest(
            content = Content(listOf(TextPart(searchQuery)))
        )
    ).embedding.vector

    val spec = SearchSpec.Builder()
        .addEmbeddingParameters(EmbeddingVector(qVec, embedModel.getModelSignature()))
        .setDefaultEmbeddingSearchMetricType(SearchSpec.EMBEDDING_SEARCH_METRIC_TYPE_COSINE)
        .setRankingStrategy("sum(this.matchedSemanticScores(getEmbeddingParameter(0)))")
        .setListFilterQueryLanguageEnabled(true).build()

}

suspend fun executeVaultSearch(searchQuery: String): VaultMatch {
    val qVec = embedModel.generateEmbedding(
        EmbeddingRequest(
            content = Content(listOf(TextPart(searchQuery)))
        )
    ).embedding.vector

    val spec = SearchSpec.Builder()
        .addEmbeddingParameters(EmbeddingVector(qVec, embedModel.getModelSignature()))
        .setDefaultEmbeddingSearchMetricType(SearchSpec.EMBEDDING_SEARCH_METRIC_TYPE_COSINE)
        .setRankingStrategy("sum(this.matchedSemanticScores(getEmbeddingParameter(0)))")
        .setListFilterQueryLanguageEnabled(true).build()

val results = appSearchSession.search(
        "semanticSearch(getEmbeddingParameter(0), 0.68)", spec
    )
    // Cleanly close the cursor.
    return results.use { it.nextPageAsync.await().first().genericDocument.toVaultMatch() }
}

val callerVector = embedModel.generateEmbedding(
    EmbeddingRequest(
        Content(listOf(

            ImagePart(callerParcelPhotoBmp),   // brown box w/ shipping label
            TextPart(callerText)               // "Clearance fee ¥6,800 due today."

        ))
    )
).embedding.vector
val matchScore = cosineSimilarity(callerVector, vaultMatch.embedding)

val forensicRequest = generateContentRequest(
    securitySystemInstruction,
    content {
        image(callerParcelPhotoBmp)
        image(orderEmailLensBmp)
        text(forensicAuditPrompt)
    }
) {

    // Enable Gemini Nano 4 reasoning before it answers
    enableThinking = true

}

@Generable(description = "Details of a visual contradiction found during an audit")
data class VisualContradiction(
  @Guide(
    description = "The type of contradiction",
    enumValues = ["PACKAGING_MISMATCH", "STATUS_CONFLICT", "DUTY_FRAUD"],
  )
  val contradictionType: String,

  @Guide(description = "Detailed explanation of the visual contradiction")
  val explanation: String,
)

@Generable(description = "Verdict and details of a phishing audit")
data class PhishingAuditVerdict(
  @Guide(
    description = "The audit verdict",
    enumValues = ["CONFIRMED_PHISHING_SCAM", "LEGITIMATE_HOLD"],
  )
  val verdict: String,

  @Guide(description = "The fraud risk score from 0 to 100", minimum = 0.0, maximum = 100.0)
  val fraudRiskScore: Int,

  @Guide(description = "Estimated amount of money saved in USD", minimum = 0.0)
  val moneySavedUsd: Int,

  @Guide(description = "List of visual contradictions identified during audit", minItems = 0)
  val contradictions: List<VisualContradiction>,

  // @Guide(description = "Summary of the warning to be spoken to the user")
  // `@Guide` annotations are optional.
  val spokenWarningSummary: String,
)

val typedRequest = generateTypedContentRequest(forensicRequest, PhishingAuditVerdict::class)
val response = generativeModel.generateContent(typedRequest)
val verdict: PhishingAuditVerdict? = response.candidates.first().response

{
  "verdict": "CONFIRMED_PHISHING_SCAM",
  "fraudRiskScore": 98,
  "moneySavedUsd": 45,
  "contradictions": [
    { "contradictionType": "DUTY_FRAUD", "explanation": "Fee demanded, but duties were prepaid." },
    { "contradictionType": "PACKAGING_MISMATCH", "explanation": "Carton has no carrier label or tracking." },
  ],
  "spokenWarningSummary": "Scam blocked. Do not pay the 45 dollar fee. Your lens shipped duties prepaid."
}

val synthesizer = SpeechSynthesis.getClient(SpeechSynthesizerOptions.builder(Locale.US).build())
try {
    val voice = synthesizer.getSupportedVoices(Locale.US).firstOrNull()
    if (voice != null) {
        val ttsRequest = SpeechSynthesizerRequest(
            input = SpeechSynthesizerRequest.Input.Text(verdict.spokenWarningSummary, voice),
            audioAttributes = AudioAttributes.Builder()
                .setUsage(AudioAttributes.USAGE_ASSISTANT)
                .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
                .build(),
        )

        /* more code here */

    }
} finally {
    synthesizer.close() // safe here: the flow completes after DONE, STOP, or Error
}

val synthesizer = SpeechSynthesis.getClient(SpeechSynthesizerOptions.builder(Locale.US).build())
try {
    val voice = synthesizer.getSupportedVoices(Locale.US).firstOrNull()
    if (voice != null) {
        val ttsRequest = SpeechSynthesizerRequest(
            input = SpeechSynthesizerRequest.Input.Text(verdict.spokenWarningSummary, voice),
            audioAttributes = AudioAttributes.Builder()
                .setUsage(AudioAttributes.USAGE_ASSISTANT)
                .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
                .build(),
        )
        synthesizer.synthesize(ttsRequest).collect { response ->
            when (response) {
                is SpeechSynthesizerResponse.StatusUpdate ->
                                            /* playback finished */
                    if (response.status == SpeechSynthesizerResponse.Status.DONE) { }

                /* response.errorCode is a TextToSpeech.ERROR_* code */
 is SpeechSynthesizerResponse.Error -> { }
                else -> {}
            }
        }
    }
} finally {
    synthesizer.close() // safe here: the flow completes after DONE, STOP, or Error
}

val synthesizer = SpeechSynthesis.getClient(SpeechSynthesizerOptions.builder(Locale.US).build())
try {
    val voice = synthesizer.getSupportedVoices(Locale.US).firstOrNull()
    if (voice != null) {
        val ttsRequest = SpeechSynthesizerRequest(
            input = SpeechSynthesizerRequest.Input.Text(verdict.spokenWarningSummary, voice),
            audioAttributes = AudioAttributes.Builder()
                .setUsage(AudioAttributes.USAGE_ASSISTANT)
                .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
                .build(),
        )
        synthesizer.synthesize(ttsRequest).collect { response ->
            when (response) {
                is SpeechSynthesizerResponse.StatusUpdate ->
                                            /* playback finished */
                    if (response.status == SpeechSynthesizerResponse.Status.DONE) { }

                /* response.errorCode is a TextToSpeech.ERROR_* code */
 is SpeechSynthesizerResponse.Error -> {
     /* handle fallback to platform API */
 }
                else -> {}
            }
        }
    }
// more code here ...