binarize handshake (#73)

hivemind_core/protocol.py

@@ -3,16 +3,17 @@
 from enum import Enum, IntEnum
 from typing import List, Dict, Optional
 
+from ovos_bus_client import MessageBusClient
+from ovos_bus_client.message import Message
+from ovos_bus_client.session import Session
 from ovos_utils.log import LOG
 from poorman_handshake import HandShake, PasswordHandShake
 from tornado import ioloop
 from tornado.websocket import WebSocketHandler
 
 from hivemind_bus_client.message import HiveMessage, HiveMessageType
-from hivemind_bus_client.util import decrypt_from_json, encrypt_as_json
-from ovos_bus_client import MessageBusClient
-from ovos_bus_client.message import Message
-from ovos_bus_client.session import Session
+from hivemind_bus_client.serialization import decode_bitstring, get_bitstring
+from hivemind_bus_client.util import decrypt_bin, encrypt_bin, decrypt_from_json, encrypt_as_json
 
 
 class ProtocolVersion(IntEnum):
@@ -53,8 +54,9 @@ class HiveMindClientConnection:
     pswd_handshake: Optional[PasswordHandShake] = None
     socket: Optional[WebSocketHandler] = None
     crypto_key: Optional[str] = None
-    blacklist: List[str] = field(default_factory=list) # list of ovos message_type to never be sent to this client
-    allowed_types: List[str] = field(default_factory=list) # list of ovos message_type to allow to be sent from this client
+    blacklist: List[str] = field(default_factory=list)  # list of ovos message_type to never be sent to this client
+    allowed_types: List[str] = field(default_factory=list)  # list of ovos message_type to allow to be sent from this client
+    binarize: bool = False
 
     @property
     def peer(self) -> str:
@@ -76,25 +78,36 @@ def send(self, message: HiveMessage):
         LOG.info(f"sending to {self.peer}: {message}")
         payload = message.serialize()  # json string
         if self.crypto_key and message.msg_type not in [HiveMessageType.HANDSHAKE,
-                                                    HiveMessageType.HELLO]:
-            payload = encrypt_as_json(self.crypto_key, payload)  # still a json string
+                                                        HiveMessageType.HELLO]:
+            if self.binarize:
+                payload = get_bitstring(message.msg_type, message.payload)
+                payload = encrypt_bin(self.crypto_key, payload)
+            else:
+                payload = encrypt_as_json(self.crypto_key, payload)  # still a json string
             LOG.info(f"encrypted payload: {len(payload)}")
         else:
-            LOG.warning(f"sent unencrypted!")
+            LOG.debug(f"sent unencrypted!")
 
         self.loop.install()
         self.socket.write_message(payload)
 
     def decode(self, payload: str) -> HiveMessage:
         if self.crypto_key:
-            if "ciphertext" in payload:
+            # handle binary encryption
+            if isinstance(payload, bytes):
+                payload = decrypt_bin(self.crypto_key, payload)
+            # handle json encryption
+            elif "ciphertext" in payload:
                 payload = decrypt_from_json(self.crypto_key, payload)
             else:
                 LOG.warning("Message was unencrypted")
                 # TODO - some error if crypto is required
         else:
             pass  # TODO - reject anything except HELLO and HANDSHAKE
-        if isinstance(payload, str):
+
+        if isinstance(payload, bytes):
+            return decode_bitstring(payload)
+        elif isinstance(payload, str):
             payload = json.loads(payload)
         return HiveMessage(**payload)
 
@@ -233,7 +246,8 @@ def handle_new_client(self, client: HiveMindClientConnection):
         max_version = ProtocolVersion.ONE
 
         msg = HiveMessage(HiveMessageType.HELLO,
-                          payload={"pubkey": client.handshake.pubkey, # allows any node to verify messages are signed with this
+                          payload={"pubkey": client.handshake.pubkey,
+                                   # allows any node to verify messages are signed with this
                                    "peer": client.peer,  # this identifies the connected client in ovos message.context
                                    "node_id": self.peer})
         LOG.info(f"saying HELLO to: {client.peer}")
@@ -246,6 +260,7 @@ def handle_new_client(self, client: HiveMindClientConnection):
             "handshake": needs_handshake,  # tell the client it must do a handshake or connection will be dropped
             "min_protocol_version": min_version,
             "max_protocol_version": max_version,
+            "binarize": True,  # report we support the binarization scheme
             "preshared_key": client.crypto_key is not None,  # do we have a pre-shared key (V0 proto)
             "password": client.pswd_handshake is not None,  # is password available (V1 proto, replaces pre-shared key)
             "crypto_required": self.require_crypto  # do we allow unencrypted payloads
@@ -307,6 +322,8 @@ def handle_message(self, message: HiveMessage, client: HiveMindClientConnection)
             self.handle_broadcast_message(message, client)
         elif message.msg_type == HiveMessageType.ESCALATE:
             self.handle_escalate_message(message, client)
+        elif message.msg_type == HiveMessageType.BINARY:
+            self.handle_binary_message(message, client)
         else:
             self.handle_unknown_message(message, client)
 
@@ -318,6 +335,9 @@ def handle_unknown_message(self, message: HiveMessage, client: HiveMindClientCon
         message (HiveMessage): HiveMind message object
         """
 
+    def handle_binary_message(self, message: HiveMessage, client: HiveMindClientConnection):
+        assert message.msg_type == HiveMessageType.BINARY
+
     def handle_handshake_message(self, message: HiveMessage,
                                  client: HiveMindClientConnection):
         LOG.info("handshake received, generating session key")
@@ -338,6 +358,7 @@ def handle_handshake_message(self, message: HiveMessage,
         elif client.pswd_handshake is not None and "envelope" in payload:
             # while the access key is transmitted, the password never is
             envelope = payload["envelope"]
+            client.binarize = payload.get("binarize", False)
 
             payload["envelope"] = client.pswd_handshake.generate_handshake()
 
@@ -348,7 +369,6 @@ def handle_handshake_message(self, message: HiveMessage,
             #     client.socket.close()
             #     return
 
-
             # key is derived safely from password in both sides
             # the handshake is validating both ends have the same password
             # the key is never actually transmitted
@@ -366,9 +386,6 @@ def handle_handshake_message(self, message: HiveMessage,
         msg = HiveMessage(HiveMessageType.HANDSHAKE, payload)
         client.send(msg)  # client can recreate crypto_key on his side now
 
-    def handle_binary_message(self, message: bytes, client: HiveMindClientConnection):
-        pass  # TODO -  binary https://github.com/JarbasHiveMind/hivemind_websocket_client/pull/4
-
     def handle_bus_message(self, message: HiveMessage,
                            client: HiveMindClientConnection):
         self.handle_inject_mycroft_msg(message.payload, client)
@@ -468,7 +485,7 @@ def handle_inject_mycroft_msg(self, message: Message, client: HiveMindClientConn
         message.context["session"] = client.sess.serialize()
         if message.msg_type == "speak":
             message.context["destination"] = ["audio"]
-        elif message.context.get("destination") is None:    
+        elif message.context.get("destination") is None:
             message.context["destination"] = "skills"  # ensure not treated as a broadcast
 
         # send client message to internal mycroft bus

hivemind_core/scripts.py

@@ -128,7 +128,7 @@ def list_clients():
     table.add_column("ID", justify="center")
     table.add_column("Name", justify="center")
     table.add_column("Access Key", justify="center")
-    table.add_column("Passwordy", justify="center")
+    table.add_column("Password", justify="center")
     table.add_column("Crypto Key", justify="center")
 
     with ClientDatabase() as db:

hivemind_core/service.py

@@ -110,13 +110,9 @@ def on(self, event_name, handler):
         self.emitter.on(event_name, handler)
 
     def on_message(self, message):
-        if isinstance(message, bytes):
-            LOG.info(f"received binary data: {len(message)}")
-            self.protocol.handle_binary_message(message, self.client)
-        else:
-            message = self.client.decode(message)
-            LOG.info(f"received {self.client.peer} message: {message}")
-            self.protocol.handle_message(message, self.client)
+        message = self.client.decode(message)
+        LOG.info(f"received {self.client.peer} message: {message}")
+        self.protocol.handle_message(message, self.client)
 
     def open(self):
         auth = self.request.uri.split("/?authorization=")[-1]

readme.md

@@ -72,6 +72,20 @@ Options:
 
 ```
 
+# Protocol
+
+| Protocol Version     | 0   | 1   |
+|----------------------|-----|-----|
+| json serialization   | yes | yes |
+| binary serialization | no  | yes |
+| pre-shared AES key   | yes | yes |
+| password handshake   | no  | yes |
+| PGP handshake        | no  | yes |
+| zlib compression     | no  | yes |
+
+
+some clients such as HiveMind-Js do not yet support protocol V1
+
 # HiveMind components
 
 ![](./resources/1m5s.svg)