#1 Emit EOF token at the end

src/AerialShip/Lex/EOF.php

@@ -0,0 +1,12 @@
+<?php
+
+namespace AerialShip\Lex;
+
+
+class EOF extends Token
+{
+    public function __construct($offset, $count)
+    {
+        parent::__construct(null, null, $offset, $count);
+    }
+}

src/AerialShip/Lex/Lexer.php

@@ -51,6 +51,7 @@ public function tokenizeAsync($input, callable $callback)
                 throw new UnknownTokenException($offset);
             }
         }
+        call_user_func($callback, new EOF($offset, $count));
     }
 
 

src/AerialShip/Lex/Tests/LexerFunctionalTest.php

@@ -84,7 +84,7 @@ public function shouldFailAtBadOffsetAsync()
     protected function assertTokens($arr)
     {
         $this->assertInternalType('array', $arr);
-        $this->assertCount(5, $arr);
+        $this->assertCount(6, $arr);
         $this->assertContainsOnlyInstancesOf('AerialShip\Lex\Token', $arr);
 
         $this->assertEquals('number', $arr[0]->getToken());
@@ -111,5 +111,9 @@ protected function assertTokens($arr)
         $this->assertEquals('567', $arr[4]->getValue());
         $this->assertEquals(15, $arr[4]->getOffset());
         $this->assertEquals(4, $arr[4]->getCount());
+
+        $this->assertInstanceOf('AerialShip\Lex\EOF', $arr[5]);
+        $this->assertEquals(18, $arr[5]->getOffset());
+        $this->assertEquals(5, $arr[5]->getCount());
     }
 }