ocr: Fix screen content reset mechanism

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.5] - 2024-04-09
+
+### Fixed
+
+- Fix OCR screen content reset mechanism
+- 
 ## [0.8.4] - 2024-04-09
 
 ### Fixed

speculos/api/events.py

@@ -9,9 +9,6 @@
 from speculos.observer import BroadcastInterface, ObserverInterface, TextEvent
 from .restful import AppResource
 
-# Approximative minimum vertical distance between two lines of text on the devices' screen.
-MIN_LINES_HEIGHT_DISTANCE = 10  # pixels
-
 
 class EventsBroadcaster(BroadcastInterface):
     """This used to be the 'Automation Server'."""
@@ -28,11 +25,9 @@ def clear_events(self) -> None:
         self.screen_content = []
 
     def broadcast(self, event: TextEvent) -> None:
-        if self.screen_content:
-            # Reset screen content if new event is not below the last text line of
-            # current screen. Event Y coordinate starts at the top of the screen.
-            if event.y <= self.screen_content[-1].y + MIN_LINES_HEIGHT_DISTANCE:
-                self.clear_events()
+        if event.clear:
+            self.clear_events()
+            return
 
         self.logger.debug("events: broadcasting %s to %d client(s)", asdict(event), len(self.clients))
         self.screen_content.append(event)

speculos/mcu/bagl.py

@@ -417,7 +417,8 @@ def _display_bagl_icon(self, component, context: bytes) -> None:
                                              bpp,
                                              bitmap)
 
-    def _display_bagl_rectangle(self, component, context, context_encoding, halignment, valignment):
+    def _display_bagl_rectangle(self, component, context, context_encoding, halignment, valignment) -> List[TextEvent]:
+        ret = []
         radius = min(component.radius, min(component.width//2, component.height//2))
         if component.fill != BAGL_FILL:
             coords = [
@@ -450,7 +451,10 @@ def _display_bagl_rectangle(self, component, context, context_encoding, halignme
                 (component.x+component.width-radius, component.y+radius, radius, component.height-2*radius),
             ]
             for (x, y, width, height) in coords:
-                self._hal_draw_rect(component.fgcolor, x, y, width, height)
+                if x == 0 and y == 0 and width == self.SCREEN_WIDTH and height == self.SCREEN_HEIGHT:
+                    ret += self.fb.draw_rect(x, y, width, height, component.fgcolor)
+                else:
+                    self._hal_draw_rect(component.fgcolor, x, y, width, height)
 
         if radius > 1:
             radiusint = 0
@@ -499,6 +503,8 @@ def _display_bagl_rectangle(self, component, context, context_encoding, halignme
                               component.height - valignment - stroke,
                               context)
 
+        return ret
+
     def _display_bagl_labeline(self,
                                component,
                                text,
@@ -540,7 +546,7 @@ def _display_bagl_labeline(self,
 
         return [TextEvent(text.decode("utf-8", "ignore"),
                           component.x + halignment, y,
-                          component.width - halignment, component.height)]
+                          component.width - halignment, component.height, False)]
 
     def _display_get_alignment(self, component, context, context_encoding):
         halignment = 0
@@ -600,7 +606,7 @@ def display_status(self, data: bytes) -> List[TextEvent]:
             # self.renderer.clear()
             pass
         elif type_ == BAGL_RECTANGLE:
-            self._display_bagl_rectangle(component, context, context_encoding, halignment, valignment)
+            ret = self._display_bagl_rectangle(component, context, context_encoding, halignment, valignment)
         elif type_ == BAGL_LABEL:
             self._display_bagl_labeline(component, context, halignment, valignment, baseline, char_height,
                                         strwidth, type_)

speculos/mcu/display.py

@@ -105,20 +105,22 @@ def draw_horizontal_line(self, x0: int, y: int, width: int, color: int) -> None:
         for x in range(x0, x0 + width):
             self.pixels[(x, y)] = self.check_color(color)
 
-    def draw_rect(self, x0: int, y0: int, width: int, height: int, color: int) -> None:
+    def draw_rect(self, x0: int, y0: int, width: int, height: int, color: int) -> List[TextEvent]:
         color = self.check_color(color)
 
         if x0 == 0 and y0 == 0 and width == self._width and height == self._height:
             self.default_color = color
             self.draw_default_color = True
             self.pixels = {}
             self.screenshot_pixels = {}
-            return
+            return [TextEvent("", 0, 0, 0, 0, True)]
 
         for x in range(x0, x0 + width):
             for y in range(y0, y0 + height):
                 self.pixels[(x, y)] = color
 
+        return []
+
     def _get_image(self) -> bytes:
         data = bytearray(self.default_color.to_bytes(3, "big")) * self._width * self._height
         for (x, y), color in self.screenshot_pixels.items():

speculos/mcu/nbgl.py

@@ -3,13 +3,14 @@
 import sys
 from construct import Struct, Int8ul, Int16ul
 from enum import IntEnum
+from speculos.observer import TextEvent
 try:
     from functools import cache
 except ImportError:
     # `functools.cache` does not exists on Python3.8
     from functools import lru_cache
     cache = lru_cache(maxsize=None)
-from typing import Tuple
+from typing import List, Tuple
 
 from .display import FrameBuffer, GraphicLibrary
 # This is a copy - original version is in the SDK (tools/rle_custom.py)
@@ -61,10 +62,10 @@ def __assert_area(self, area) -> None:
         if area.y0 > self.SCREEN_HEIGHT or (area.y0+area.height) > self.SCREEN_HEIGHT:
             raise AssertionError("top edge (%d) or bottom edge (%d) out of screen" % (area.y0, (area.y0 + area.height)))
 
-    def hal_draw_rect(self, data: bytes) -> None:
+    def hal_draw_rect(self, data: bytes) -> List[TextEvent]:
         area = nbgl_area_t.parse(data)
         self.__assert_area(area)
-        self.fb.draw_rect(area.x0, area.y0, area.width, area.height, NBGL.to_screen_color(area.color, 2))
+        return self.fb.draw_rect(area.x0, area.y0, area.width, area.height, NBGL.to_screen_color(area.color, 2))
 
     def refresh(self, data: bytes) -> bool:
         area = nbgl_area_t.parse(data)

speculos/mcu/ocr.py

@@ -170,7 +170,7 @@ def find_bitmap(self, x: int, y: int, w: int, h: int, bitmap: bytes) -> None:
             else:
                 # create a new TextEvent if there are no events yet
                 # or if there is a new line
-                self.events.append(TextEvent(char, x, y, w, h))
+                self.events.append(TextEvent(char, x, y, w, h, False))
 
     def store_char_in_last_event(self, x: int, y: int, w: int, h: int, char: str) -> None:
         """
@@ -210,7 +210,7 @@ def add_character(self, x: int, y: int, w: int, h: int, char: str) -> None:
                 return
 
         # create a new TextEvent if there are no events yet or if there is a new line
-        self.events.append(TextEvent(char, x, y, w, h))
+        self.events.append(TextEvent(char, x, y, w, h, False))
 
     def analyze_bitmap(self, data: bytes) -> None:
         """

speculos/mcu/seproxyhal.py

@@ -353,8 +353,7 @@ def can_read(self, screen: DisplayNotifier):
             self.logger.debug(f"DISPLAY_STATUS {data!r}")
             if screen.display.model not in ["nanox", "nanosp"] or tag == SephTag.BAGL_DRAW_RECT:
                 events = screen.display.display_status(data)
-                if events:
-                    self.events += events
+                self.events += events
             if tag != SephTag.BAGL_DRAW_RECT:
                 self.socket_helper.send_packet(SephTag.DISPLAY_PROCESSED_EVENT)
 
@@ -416,7 +415,7 @@ def can_read(self, screen: DisplayNotifier):
 
         elif tag == SephTag.NBGL_DRAW_RECT:
             assert isinstance(screen.display.gl, NBGL)
-            screen.display.gl.hal_draw_rect(data)
+            self.events += screen.display.gl.hal_draw_rect(data)
 
         elif tag == SephTag.NBGL_REFRESH:
             assert isinstance(screen.display.gl, NBGL)

speculos/observer.py

@@ -11,6 +11,7 @@ class TextEvent:
     y: int
     w: int
     h: int
+    clear: bool
 
 
 class ObserverInterface(ABC):

tests/python/api/test_api.py

@@ -98,11 +98,11 @@ def get_current_screen_content(session):
                         assert re.match(text, event["text"])
 
             texts = [
-                '{"events": [{"text": "Bitcoin", "x": 41, "y": 3, "w": 128, "h": 32}, '
-                '{"text": "is ready", "x": 41, "y": 17, "w": 128, "h": 32}]}\n',
-                '{"events": [{"text": "Version", "x": 43, "y": 3, "w": 79, "h": 32}, '
-                '{"text": "2.0.1", "x": 52, "y": 17, "w": 70, "h": 32}]}\n',
-                '{"events": [{"text": "About", "x": 47, "y": 19, "w": 81, "h": 32}]}\n'
+                '{"events": [{"text": "Bitcoin", "x": 41, "y": 3, "w": 128, "h": 32, "clear": false}, '
+                '{"text": "is ready", "x": 41, "y": 17, "w": 128, "h": 32, "clear": false}]}\n',
+                '{"events": [{"text": "Version", "x": 43, "y": 3, "w": 79, "h": 32, "clear": false}, '
+                '{"text": "2.0.1", "x": 52, "y": 17, "w": 70, "h": 32, "clear": false}]}\n',
+                '{"events": [{"text": "About", "x": 47, "y": 19, "w": 81, "h": 32, "clear": false}]}\n'
             ]
             for text in texts:
                 content = get_current_screen_content(r)