POC for integrating Remote web server with Openbot (#369)

### Summary of Changes #### Code Improvements - **Bug Fixes**: - Fixed syntax errors in JS generator - Resolved frame parameter issues - Updated object type handling - Addressed navigation issues in Blockly - Managed token expiration handling - **New Features**: - Added dynamic speed UI for object tracking - Enabled local data saving for controllers and sensor statuses - Integrated autopilot parameters - Implemented camera switch status - Introduced web server control and text-to-speech functionalities - **Code Refactoring**: - Removed unused variables, logs, and redundant files - Optimized shared preferences and documentation alignment #### Documentation - **README.md Updates**: - Added details for Google Drive API, web controller, AutoSync, and node.js controller - Included new images, gifs, and fixed links - Improved alignment and styling
isl-org · Jul 4, 2024 · 65b158d · 65b158d
1 parent caeba05
commit 65b158d
Show file tree

Hide file tree

Showing 96 changed files with 21,378 additions and 156 deletions.
diff --git a/android/robot/build.gradle b/android/robot/build.gradle
@@ -175,6 +175,9 @@ dependencies {
     implementation 'com.github.Ficat:EasyBle:v2.0.2'
     implementation 'com.github.Ficat:EasyPermissions:v2.1.0'
 
+    // Socket Server
+    implementation 'com.squareup.okhttp3:okhttp:4.9.1'
+
     //Scan QR
     implementation 'com.google.android.gms:play-services-vision:20.1.3'
     implementation('com.google.api-client:google-api-client-android:1.31.0') {

diff --git a/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java b/android/robot/src/main/java/org/openbot/autopilot/AutopilotFragment.java
@@ -16,19 +16,15 @@
 import android.view.ViewGroup;
 import android.widget.AdapterView;
 import android.widget.Toast;
-
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.camera.core.ImageProxy;
 import androidx.navigation.Navigation;
-
 import com.google.android.material.bottomsheet.BottomSheetBehavior;
-
 import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
-
 import org.jetbrains.annotations.NotNull;
 import org.openbot.R;
 import org.openbot.common.CameraFragment;
@@ -43,7 +39,6 @@
 import org.openbot.utils.Enums;
 import org.openbot.utils.PermissionUtils;
 import org.openbot.vehicle.Control;
-
 import timber.log.Timber;
 
 public class AutopilotFragment extends CameraFragment {
@@ -461,6 +456,16 @@ protected Model getModel() {
         return model;
     }
 
+    private void connectWebController() {
+        phoneController.connectWebServer();
+        Enums.DriveMode oldDriveMode = currentDriveMode;
+        // Currently only dual drive mode supported
+        setDriveMode(Enums.DriveMode.GAME);
+        binding.controllerContainer.driveMode.setAlpha(0.5f);
+        binding.controllerContainer.driveMode.setEnabled(false);
+        preferencesManager.setDriveMode(oldDriveMode.getValue());
+    }
+
     protected void setModel(Model model) {
         if (this.model != model) {
             Timber.d("Updating  model: %s", model);
@@ -534,7 +539,12 @@ private void setControlMode(Enums.ControlMode controlMode) {
                     if (!PermissionUtils.hasControllerPermissions(requireActivity()))
                         requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
                     else connectPhoneController();
-
+                    break;
+                case WEBSERVER:
+                    binding.controllerContainer.controlMode.setImageResource(R.drawable.ic_server);
+                    if (!PermissionUtils.hasControllerPermissions(requireActivity()))
+                        requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
+                    else connectWebController();
                     break;
             }
             Timber.d("Updating  controlMode: %s", controlMode);
@@ -579,3 +589,4 @@ private void disconnectPhoneController() {
         binding.controllerContainer.driveMode.setAlpha(1.0f);
     }
 }
+
diff --git a/android/robot/src/main/java/org/openbot/env/PhoneController.java b/android/robot/src/main/java/org/openbot/env/PhoneController.java
@@ -2,15 +2,29 @@
 
 import android.app.Activity;
 import android.content.Context;
+import android.os.Handler;
+import android.os.Looper;
 import android.util.Log;
 import android.view.View;
 import android.view.ViewGroup;
 import android.widget.LinearLayout;
+
+import androidx.annotation.NonNull;
+
+import com.google.firebase.auth.FirebaseAuth;
+
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.openbot.R;
 import org.openbot.customview.AutoFitSurfaceGlView;
 import org.openbot.customview.WebRTCSurfaceView;
 import org.openbot.utils.CameraUtils;
+
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.WebSocket;
+import okhttp3.WebSocketListener;
+import okio.ByteString;
 import timber.log.Timber;
 
 @SuppressWarnings("ResultOfMethodCallIgnored")
@@ -20,6 +34,7 @@ public class PhoneController {
   private ConnectionSelector connectionSelector;
   private IVideoServer videoServer;
   private View view = null;
+  private WebSocket webSocket;
 
   public static PhoneController getInstance(Context context) {
     if (_phoneController == null) { // Check for the first time
@@ -104,12 +119,69 @@ public void connect(Context context) {
     }
   }
 
+  public void connectWebServer(){
+    nodeServerConnect();
+  }
+
+  private void nodeServerConnect() {
+//     String serverUrl = "ws://verdant-imported-peanut.glitch.me";
+           String serverUrl = "ws://192.168.1.6:8080";
+
+    OkHttpClient client = new OkHttpClient();
+    Request request = new Request.Builder().url(serverUrl).build();
+
+    WebSocketListener webSocketListener = new WebSocketListener() {
+      @Override
+      public void onOpen(WebSocket webSocket, @NonNull okhttp3.Response response) {
+        ControllerToBotEventBus.emitEvent("{command: \"CONNECTED\"}");
+      }
+
+      @Override
+      public void onMessage(@NonNull WebSocket webSocket, @NonNull String text) {
+        // Called when text message is received from the server
+        ControllerToBotEventBus.emitEvent(text);
+      }
+
+      @Override
+      public void onMessage(@NonNull WebSocket webSocket, @NonNull ByteString bytes) {
+        // Called when binary message is received from the server
+      }
+
+      @Override
+      public void onClosing(@NonNull WebSocket webSocket, int code, @NonNull String reason) {
+        // Called when the connection is closing
+      }
+
+      @Override
+      public void onClosed(@NonNull WebSocket webSocket, int code, @NonNull String reason) {
+        // Called when the connection is closed
+      }
+
+      @Override
+      public void onFailure(@NonNull WebSocket webSocket, Throwable t, okhttp3.Response response) {
+        // Called when an error occurs
+      }
+    };
+
+    webSocket = client.newWebSocket(request, webSocketListener);
+  }
+
   public void disconnect() {
     connectionSelector.getConnection().stop();
   }
 
   public void send(JSONObject info) {
-    connectionSelector.getConnection().sendMessage(info.toString());
+    if (webSocket != null && FirebaseAuth.getInstance().getCurrentUser() != null)
+      try {
+        info.put("roomId", FirebaseAuth.getInstance().getCurrentUser().getEmail()); // Add the roomId to the JSON object
+        String messageString = info.toString();
+        webSocket.send(messageString);
+      } catch (JSONException e) {
+        throw new RuntimeException(e);
+      }
+
+     if (connectionSelector.getConnection().isConnected())
+       connectionSelector.getConnection().sendMessage(info.toString());
   }
 
   public boolean isConnected() {
@@ -127,11 +199,13 @@ private void monitorConnection() {
         event -> {
           switch (event.getString("command")) {
             case "CONNECTED":
-              videoServer.setConnected(true);
+              new Handler(Looper.getMainLooper()).post(() -> videoServer.setConnected(true));
+//              videoServer.setConnected(true);
               break;
 
             case "DISCONNECTED":
-              videoServer.setConnected(false);
+              new Handler(Looper.getMainLooper()).post(() -> videoServer.setConnected(false));
+//              videoServer.setConnected(false);
               break;
           }
         },

diff --git a/android/robot/src/main/java/org/openbot/logging/LoggerFragment.java b/android/robot/src/main/java/org/openbot/logging/LoggerFragment.java
@@ -551,6 +551,12 @@ private void setControlMode(Enums.ControlMode controlMode) {
             requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
           else connectPhoneController();
           break;
+        case WEBSERVER:
+          binding.controllerContainer.controlMode.setImageResource(R.drawable.ic_server);
+          if (!PermissionUtils.hasControllerPermissions(requireActivity()))
+            requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
+          else connectWebController();
+          break;
       }
       Timber.d("Updating  controlMode: %s", controlMode);
       preferencesManager.setControlMode(controlMode.getValue());
@@ -587,6 +593,16 @@ private void connectPhoneController() {
     preferencesManager.setDriveMode(oldDriveMode.getValue());
   }
 
+  private void connectWebController() {
+    phoneController.connectWebServer();
+    Enums.DriveMode oldDriveMode = currentDriveMode;
+    // Currently only dual drive mode supported
+    setDriveMode(Enums.DriveMode.GAME);
+    binding.controllerContainer.driveMode.setAlpha(0.5f);
+    binding.controllerContainer.driveMode.setEnabled(false);
+    preferencesManager.setDriveMode(oldDriveMode.getValue());
+  }
+
   private void disconnectPhoneController() {
     phoneController.disconnect();
     setDriveMode(Enums.DriveMode.getByID(preferencesManager.getDriveMode()));

diff --git a/android/robot/src/main/java/org/openbot/modelManagement/ModelManagementFragment.java b/android/robot/src/main/java/org/openbot/modelManagement/ModelManagementFragment.java
@@ -22,7 +22,6 @@
 import android.widget.AdapterView;
 import android.widget.ArrayAdapter;
 import android.widget.Toast;
-
 import androidx.activity.OnBackPressedCallback;
 import androidx.activity.result.ActivityResultLauncher;
 import androidx.activity.result.contract.ActivityResultContracts;
@@ -38,15 +37,13 @@
 import com.google.firebase.auth.FirebaseUser;
 import com.google.gson.Gson;
 import com.nononsenseapps.filepicker.Utils;
-
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
-
 import org.openbot.R;
 import org.openbot.databinding.FragmentModelManagementBinding;
 import org.openbot.googleServices.GoogleServices;

diff --git a/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java b/android/robot/src/main/java/org/openbot/objectNav/ObjectNavFragment.java
@@ -617,7 +617,12 @@ private void setControlMode(Enums.ControlMode controlMode) {
           if (!PermissionUtils.hasControllerPermissions(requireActivity()))
             requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
           else connectPhoneController();
-
+          break;
+        case WEBSERVER:
+          binding.controllerContainer.controlMode.setImageResource(R.drawable.ic_server);
+          if (!PermissionUtils.hasControllerPermissions(requireActivity()))
+            requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
+          else connectWebController();
           break;
       }
       Timber.d("Updating  controlMode: %s", controlMode);
@@ -655,6 +660,16 @@ private void connectPhoneController() {
     preferencesManager.setDriveMode(oldDriveMode.getValue());
   }
 
+  private void connectWebController() {
+    phoneController.connectWebServer();
+    Enums.DriveMode oldDriveMode = currentDriveMode;
+    // Currently only dual drive mode supported
+    setDriveMode(Enums.DriveMode.GAME);
+    binding.controllerContainer.driveMode.setAlpha(0.5f);
+    binding.controllerContainer.driveMode.setEnabled(false);
+    preferencesManager.setDriveMode(oldDriveMode.getValue());
+  }
+
   private void disconnectPhoneController() {
     phoneController.disconnect();
     setDriveMode(Enums.DriveMode.getByID(preferencesManager.getDriveMode()));

diff --git a/android/robot/src/main/java/org/openbot/original/CameraActivity.java b/android/robot/src/main/java/org/openbot/original/CameraActivity.java
@@ -920,7 +920,11 @@ private void setControlMode(ControlMode controlMode) {
             PermissionUtils.requestControllerPermissions(this);
           } else connectPhoneController();
           break;
-        case WEBRTC:
+        case WEBSERVER:
+          handleControllerEvents();
+          if (!PermissionUtils.hasControllerPermissions(this)) {
+            PermissionUtils.requestControllerPermissions(this);
+          } else connectWebController();
           break;
         default:
           throw new IllegalStateException("Unexpected value: " + controlMode);
@@ -939,6 +943,15 @@ private void connectPhoneController() {
     preferencesManager.setDriveMode(oldDriveMode.getValue());
   }
 
+  private void connectWebController() {
+    phoneController.connectWebServer();
+    DriveMode oldDriveMode = driveMode;
+    // Currently only dual drive mode supported
+    setDriveMode(Enums.DriveMode.GAME);
+    driveModeSpinner.setAlpha(0.5f);
+    preferencesManager.setDriveMode(oldDriveMode.getValue());
+  }
+
   private void disconnectPhoneController() {
     phoneController.disconnect();
     setDriveMode(DriveMode.values()[preferencesManager.getDriveMode()]);

diff --git a/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java b/android/robot/src/main/java/org/openbot/robot/FreeRoamFragment.java
@@ -226,6 +226,14 @@ private void setControlMode(ControlMode controlMode) {
             requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
           else connectPhoneController();
           break;
+        case WEBSERVER:
+          binding.controllerContainer.controlMode.setImageResource(R.drawable.ic_server);
+          if (!PermissionUtils.hasControllerPermissions(requireActivity()))
+            requestPermissionLauncher.launch(Constants.PERMISSIONS_CONTROLLER);
+          else {
+            connectWebController();
+          }
+          break;
       }
       Timber.d("Updating  controlMode: %s", controlMode);
       preferencesManager.setControlMode(controlMode.getValue());
@@ -262,6 +270,16 @@ private void connectPhoneController() {
     preferencesManager.setDriveMode(oldDriveMode.getValue());
   }
 
+  private void connectWebController() {
+    phoneController.connectWebServer();
+    Enums.DriveMode oldDriveMode = currentDriveMode;
+    // Currently only dual drive mode supported
+    setDriveMode(Enums.DriveMode.GAME);
+    binding.controllerContainer.driveMode.setAlpha(0.5f);
+    binding.controllerContainer.driveMode.setEnabled(false);
+    preferencesManager.setDriveMode(oldDriveMode.getValue());
+  }
+
   private void disconnectPhoneController() {
     phoneController.disconnect();
     setDriveMode(DriveMode.getByID(preferencesManager.getDriveMode()));

diff --git a/android/robot/src/main/java/org/openbot/server/ServerCommunication.java b/android/robot/src/main/java/org/openbot/server/ServerCommunication.java
@@ -212,7 +212,6 @@ public void upload(File file) {
       Timber.e("File not found: %s", file.getAbsolutePath());
       return;
     }
-    System.out.println("url is: " + serverUrl);
     Timber.e("Url is" + serverUrl + "/upload" );
     Timber.e("paramerter is ->" + params );
     client.post(context, serverUrl + "/upload", params, new UploadResponseHandler(file));

diff --git a/android/robot/src/main/java/org/openbot/utils/Enums.java b/android/robot/src/main/java/org/openbot/utils/Enums.java
@@ -49,8 +49,7 @@ public int getValue() {
   public enum ControlMode {
     GAMEPAD(0),
     PHONE(1),
-    WEBRTC(2);
-
+    WEBSERVER(2);
     private final int value;
 
     ControlMode(final int value) {
@@ -76,6 +75,8 @@ public static ControlMode switchControlMode(ControlMode mode) {
       case GAMEPAD:
         return ControlMode.PHONE;
       case PHONE:
+        return ControlMode.WEBSERVER;
+      case WEBSERVER:
         return ControlMode.GAMEPAD;
     }
     return null;

diff --git a/android/robot/src/main/java/org/openbot/vehicle/Vehicle.java b/android/robot/src/main/java/org/openbot/vehicle/Vehicle.java
@@ -389,7 +389,6 @@ public boolean isUsbConnected() {
   }
 
   private void sendStringToDevice(String message) {
-    System.out.println("sanjeev == " + message);
     if (getConnectionType().equals("USB") && usbConnection != null) {
       usbConnection.send(message);
     } else if (getConnectionType().equals("Bluetooth")

diff --git a/android/robot/src/main/res/drawable/ic_server.xml b/android/robot/src/main/res/drawable/ic_server.xml
@@ -0,0 +1,5 @@
+<vector android:height="24dp" android:tint="#FFFFFF"
+    android:viewportHeight="24" android:viewportWidth="24"
+    android:width="24dp" xmlns:android="http://schemas.android.com/apk/res/android">
+    <path android:fillColor="@android:color/white" android:pathData="M11.99,2C6.47,2 2,6.48 2,12s4.47,10 9.99,10C17.52,22 22,17.52 22,12S17.52,2 11.99,2zM18.92,8h-2.95c-0.32,-1.25 -0.78,-2.45 -1.38,-3.56 1.84,0.63 3.37,1.91 4.33,3.56zM12,4.04c0.83,1.2 1.48,2.53 1.91,3.96h-3.82c0.43,-1.43 1.08,-2.76 1.91,-3.96zM4.26,14C4.1,13.36 4,12.69 4,12s0.1,-1.36 0.26,-2h3.38c-0.08,0.66 -0.14,1.32 -0.14,2 0,0.68 0.06,1.34 0.14,2L4.26,14zM5.08,16h2.95c0.32,1.25 0.78,2.45 1.38,3.56 -1.84,-0.63 -3.37,-1.9 -4.33,-3.56zM8.03,8L5.08,8c0.96,-1.66 2.49,-2.93 4.33,-3.56C8.81,5.55 8.35,6.75 8.03,8zM12,19.96c-0.83,-1.2 -1.48,-2.53 -1.91,-3.96h3.82c-0.43,1.43 -1.08,2.76 -1.91,3.96zM14.34,14L9.66,14c-0.09,-0.66 -0.16,-1.32 -0.16,-2 0,-0.68 0.07,-1.35 0.16,-2h4.68c0.09,0.65 0.16,1.32 0.16,2 0,0.68 -0.07,1.34 -0.16,2zM14.59,19.56c0.6,-1.11 1.06,-2.31 1.38,-3.56h2.95c-0.96,1.65 -2.49,2.93 -4.33,3.56zM16.36,14c0.08,-0.66 0.14,-1.32 0.14,-2 0,-0.68 -0.06,-1.34 -0.14,-2h3.38c0.16,0.64 0.26,1.31 0.26,2s-0.1,1.36 -0.26,2h-3.38z"/>
+</vector>