blob: 96f4a01f7f3a48823f5cd0635338c39d20c0ad51 [file] [log] [blame]
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.speech;
import static com.android.internal.infra.AbstractRemoteService.PERMANENT_BOUND_TIMEOUT_MS;
import android.annotation.NonNull;
import android.content.AttributionSource;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.os.Bundle;
import android.os.IBinder;
import android.os.RemoteException;
import android.speech.IModelDownloadListener;
import android.speech.IRecognitionListener;
import android.speech.IRecognitionService;
import android.speech.IRecognitionSupportCallback;
import android.speech.RecognitionService;
import android.speech.SpeechRecognizer;
import android.text.TextUtils;
import android.util.Log;
import android.util.Pair;
import android.util.Slog;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.infra.ServiceConnector;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecognitionService> {
private static final String TAG = RemoteSpeechRecognitionService.class.getSimpleName();
private static final boolean DEBUG = false;
/** Maximum number of clients connected to this object at the same time. */
private static final int MAX_CONCURRENT_CLIENTS = 100;
private final Object mLock = new Object();
private boolean mConnected = false;
/** Map containing info about connected clients indexed by the their listeners. */
@GuardedBy("mLock")
private final Map<IBinder, ClientState> mClients = new HashMap<>();
/** List of pairs associating clients' binder tokens with corresponding listeners. */
@GuardedBy("mLock")
private final List<Pair<IBinder, IRecognitionListener>> mClientListeners = new ArrayList<>();
private final int mCallingUid;
private final ComponentName mComponentName;
RemoteSpeechRecognitionService(
Context context, ComponentName serviceName, int userId, int callingUid) {
super(context,
new Intent(RecognitionService.SERVICE_INTERFACE).setComponent(serviceName),
Context.BIND_AUTO_CREATE
| Context.BIND_FOREGROUND_SERVICE
| Context.BIND_INCLUDE_CAPABILITIES,
userId,
IRecognitionService.Stub::asInterface);
mCallingUid = callingUid;
mComponentName = serviceName;
if (DEBUG) {
Slog.i(TAG, "Bound to recognition service at: " + serviceName.flattenToString() + ".");
}
}
ComponentName getServiceComponentName() {
return mComponentName;
}
void startListening(Intent recognizerIntent, IRecognitionListener listener,
@NonNull AttributionSource attributionSource) {
if (DEBUG) {
Slog.i(TAG, TextUtils.formatSimple("#startListening for package: "
+ "%s, feature=%s, callingUid=%d.",
attributionSource.getPackageName(), attributionSource.getAttributionTag(),
mCallingUid));
}
if (listener == null) {
Slog.w(TAG, "#startListening called with no preceding #setListening - ignoring.");
return;
}
if (!mConnected) {
tryRespondWithError(listener, SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
return;
}
synchronized (mLock) {
ClientState clientState = mClients.get(listener.asBinder());
if (clientState == null) {
if (mClients.size() >= MAX_CONCURRENT_CLIENTS) {
tryRespondWithError(listener, SpeechRecognizer.ERROR_RECOGNIZER_BUSY);
Log.i(TAG, "#startListening received "
+ "when the recognizer's capacity is full - ignoring this call.");
return;
}
final ClientState newClientState = new ClientState();
newClientState.mDelegatingListener = new DelegatingListener(listener,
() -> {
// To be invoked in terminal calls on success.
if (DEBUG) {
Slog.i(TAG, "Recognition session completed successfully.");
}
synchronized (mLock) {
newClientState.mRecordingInProgress = false;
}
},
() -> {
// To be invoked in terminal calls on failure.
if (DEBUG) {
Slog.i(TAG, "Recognition session failed.");
}
removeClient(listener);
});
if (DEBUG) {
Log.d(TAG, "Added a new client to the map.");
}
mClients.put(listener.asBinder(), newClientState);
clientState = newClientState;
} else {
if (clientState.mRecordingInProgress) {
Slog.i(TAG, "#startListening called "
+ "while listening is in progress for this caller.");
tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
return;
}
clientState.mRecordingInProgress = true;
}
// Eager local evaluation to avoid reading a different or null value at closure runtime.
final DelegatingListener listenerToStart = clientState.mDelegatingListener;
run(service ->
service.startListening(
recognizerIntent,
listenerToStart,
attributionSource));
}
}
void stopListening(IRecognitionListener listener) {
if (DEBUG) {
Slog.i(TAG, "#stopListening");
}
if (!mConnected) {
tryRespondWithError(listener, SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
return;
}
synchronized (mLock) {
ClientState clientState = mClients.get(listener.asBinder());
if (clientState == null) {
Slog.w(TAG, "#stopListening called with no preceding #startListening - ignoring.");
tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
return;
}
if (!clientState.mRecordingInProgress) {
tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
Slog.i(TAG, "#stopListening called while listening isn't in progress - ignoring.");
return;
}
clientState.mRecordingInProgress = false;
// Eager local evaluation to avoid reading a different or null value at closure runtime.
final DelegatingListener listenerToStop = clientState.mDelegatingListener;
run(service -> service.stopListening(listenerToStop));
}
}
void cancel(IRecognitionListener listener, boolean isShutdown) {
if (DEBUG) {
Slog.i(TAG, "#cancel");
}
if (!mConnected) {
tryRespondWithError(listener, SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
}
synchronized (mLock) {
ClientState clientState = mClients.get(listener.asBinder());
if (clientState == null) {
if (DEBUG) {
Slog.w(TAG, "#cancel called with no preceding #startListening - ignoring.");
}
return;
}
clientState.mRecordingInProgress = false;
// Temporary reference to allow for resetting the hard link mDelegatingListener to null.
final IRecognitionListener delegatingListener = clientState.mDelegatingListener;
run(service -> service.cancel(delegatingListener, isShutdown));
// If shutdown, remove the client info from the map. Unbind if that was the last client.
if (isShutdown) {
removeClient(listener);
if (mClients.isEmpty()) {
if (DEBUG) {
Slog.d(TAG, "Unbinding from the recognition service.");
}
run(service -> unbind());
}
}
}
}
void checkRecognitionSupport(
Intent recognizerIntent,
AttributionSource attributionSource,
IRecognitionSupportCallback callback) {
if (!mConnected) {
try {
callback.onError(SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report the connection broke to the caller.", e);
e.printStackTrace();
}
return;
}
run(service ->
service.checkRecognitionSupport(recognizerIntent, attributionSource, callback));
}
void triggerModelDownload(
Intent recognizerIntent,
AttributionSource attributionSource,
IModelDownloadListener listener) {
if (!mConnected) {
try {
listener.onError(SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
} catch (RemoteException e) {
Slog.w(TAG, "#downloadModel failed due to connection.", e);
e.printStackTrace();
}
return;
}
run(service -> service.triggerModelDownload(recognizerIntent, attributionSource, listener));
}
void shutdown(IBinder clientToken) {
synchronized (mLock) {
for (Pair<IBinder, IRecognitionListener> clientListener : mClientListeners) {
if (clientListener.first == clientToken) {
cancel(clientListener.second, /* isShutdown */ true);
}
}
}
}
@Override // from ServiceConnector.Impl
protected void onServiceConnectionStatusChanged(
IRecognitionService service, boolean connected) {
mConnected = connected;
if (DEBUG) {
if (connected) {
Slog.i(TAG, "Connected to speech recognition service");
} else {
Slog.w(TAG, "Disconnected from speech recognition service");
}
}
synchronized (mLock) {
if (!connected) {
if (mClients.isEmpty()) {
Slog.i(TAG, "Connection to speech recognition service lost, but no "
+ "#startListening has been invoked yet.");
return;
}
for (ClientState clientState : mClients.values()) {
tryRespondWithError(
clientState.mDelegatingListener.mRemoteListener,
SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
removeClient(clientState.mDelegatingListener.mRemoteListener);
}
}
}
}
@Override // from AbstractRemoteService
protected long getAutoDisconnectTimeoutMs() {
return PERMANENT_BOUND_TIMEOUT_MS;
}
private void removeClient(IRecognitionListener listener) {
synchronized (mLock) {
ClientState clientState = mClients.remove(listener.asBinder());
if (clientState != null) {
if (DEBUG) {
Slog.d(TAG, "Removed a client from the map with listener = "
+ listener.asBinder() + ".");
}
clientState.reset();
}
mClientListeners.removeIf(clientListener -> clientListener.second == listener);
}
}
private static void tryRespondWithError(IRecognitionListener listener, int errorCode) {
if (DEBUG) {
Slog.i(TAG, "Responding with error " + errorCode);
}
try {
if (listener != null) {
listener.onError(errorCode);
}
} catch (RemoteException e) {
Slog.w(TAG,
TextUtils.formatSimple("Failed to respond with an error %d to the client",
errorCode), e);
}
}
boolean hasActiveSessions() {
synchronized (mLock) {
return !mClients.isEmpty();
}
}
void associateClientWithActiveListener(IBinder clientToken, IRecognitionListener listener) {
synchronized (mLock) {
if (mClients.containsKey(listener.asBinder())) {
mClientListeners.add(new Pair<>(clientToken, listener));
}
}
}
private static class DelegatingListener extends IRecognitionListener.Stub {
private final IRecognitionListener mRemoteListener;
private final Runnable mOnSessionSuccess;
private final Runnable mOnSessionFailure;
DelegatingListener(IRecognitionListener listener,
Runnable onSessionSuccess, Runnable onSessionFailure) {
mRemoteListener = listener;
mOnSessionSuccess = onSessionSuccess;
mOnSessionFailure = onSessionFailure;
}
@Override
public void onReadyForSpeech(Bundle params) throws RemoteException {
mRemoteListener.onReadyForSpeech(params);
}
@Override
public void onBeginningOfSpeech() throws RemoteException {
mRemoteListener.onBeginningOfSpeech();
}
@Override
public void onRmsChanged(float rmsdB) throws RemoteException {
mRemoteListener.onRmsChanged(rmsdB);
}
@Override
public void onBufferReceived(byte[] buffer) throws RemoteException {
mRemoteListener.onBufferReceived(buffer);
}
@Override
public void onEndOfSpeech() throws RemoteException {
mRemoteListener.onEndOfSpeech();
}
@Override
public void onError(int error) throws RemoteException {
if (DEBUG) {
Slog.i(TAG, TextUtils.formatSimple("Error %d during recognition session.", error));
}
mOnSessionFailure.run();
mRemoteListener.onError(error);
}
@Override
public void onResults(Bundle results) throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "#onResults invoked for a recognition session.");
}
mOnSessionSuccess.run();
mRemoteListener.onResults(results);
}
@Override
public void onPartialResults(Bundle results) throws RemoteException {
mRemoteListener.onPartialResults(results);
}
@Override
public void onSegmentResults(Bundle results) throws RemoteException {
mRemoteListener.onSegmentResults(results);
}
@Override
public void onEndOfSegmentedSession() throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "#onEndOfSegmentedSession invoked for a recognition session.");
}
mOnSessionSuccess.run();
mRemoteListener.onEndOfSegmentedSession();
}
@Override
public void onLanguageDetection(Bundle results) throws RemoteException {
mRemoteListener.onLanguageDetection(results);
}
@Override
public void onEvent(int eventType, Bundle params) throws RemoteException {
mRemoteListener.onEvent(eventType, params);
}
}
/**
* Data class holding info about a connected client:
* <ul>
* <li> {@link ClientState#mDelegatingListener}
* - object holding callbacks to be invoked after the session is complete;
* <li> {@link ClientState#mRecordingInProgress}
* - flag denoting if the client is currently recording.
*/
static class ClientState {
DelegatingListener mDelegatingListener;
boolean mRecordingInProgress;
ClientState(DelegatingListener delegatingListener, boolean recordingInProgress) {
mDelegatingListener = delegatingListener;
mRecordingInProgress = recordingInProgress;
}
ClientState(DelegatingListener delegatingListener) {
this(delegatingListener, true);
}
ClientState() {
this(null, true);
}
void reset() {
mDelegatingListener = null;
mRecordingInProgress = false;
}
}
}