From 38247ae32684396140638d0dece6803c5e319747 Mon Sep 17 00:00:00 2001
From: Milos Tomic <tmilos@gmail.com>
Date: Sun, 19 Jan 2014 14:29:07 +0100
Subject: [PATCH] #1 Emit EOF token at the end

---
 src/AerialShip/Lex/EOF.php                       | 12 ++++++++++++
 src/AerialShip/Lex/Lexer.php                     |  1 +
 src/AerialShip/Lex/Tests/LexerFunctionalTest.php |  6 +++++-
 3 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 src/AerialShip/Lex/EOF.php

diff --git a/src/AerialShip/Lex/EOF.php b/src/AerialShip/Lex/EOF.php
new file mode 100644
index 0000000..9c8d63e
--- /dev/null
+++ b/src/AerialShip/Lex/EOF.php
@@ -0,0 +1,12 @@
+<?php
+
+namespace AerialShip\Lex;
+
+
+class EOF extends Token
+{
+    public function __construct($offset, $count)
+    {
+        parent::__construct(null, null, $offset, $count);
+    }
+}
diff --git a/src/AerialShip/Lex/Lexer.php b/src/AerialShip/Lex/Lexer.php
index 0492670..45a1a47 100644
--- a/src/AerialShip/Lex/Lexer.php
+++ b/src/AerialShip/Lex/Lexer.php
@@ -51,6 +51,7 @@ public function tokenizeAsync($input, callable $callback)
                 throw new UnknownTokenException($offset);
             }
         }
+        call_user_func($callback, new EOF($offset, $count));
     }
 
 
diff --git a/src/AerialShip/Lex/Tests/LexerFunctionalTest.php b/src/AerialShip/Lex/Tests/LexerFunctionalTest.php
index 9430c6f..ce98e3e 100644
--- a/src/AerialShip/Lex/Tests/LexerFunctionalTest.php
+++ b/src/AerialShip/Lex/Tests/LexerFunctionalTest.php
@@ -84,7 +84,7 @@ public function shouldFailAtBadOffsetAsync()
     protected function assertTokens($arr)
     {
         $this->assertInternalType('array', $arr);
-        $this->assertCount(5, $arr);
+        $this->assertCount(6, $arr);
         $this->assertContainsOnlyInstancesOf('AerialShip\Lex\Token', $arr);
 
         $this->assertEquals('number', $arr[0]->getToken());
@@ -111,5 +111,9 @@ protected function assertTokens($arr)
         $this->assertEquals('567', $arr[4]->getValue());
         $this->assertEquals(15, $arr[4]->getOffset());
         $this->assertEquals(4, $arr[4]->getCount());
+
+        $this->assertInstanceOf('AerialShip\Lex\EOF', $arr[5]);
+        $this->assertEquals(18, $arr[5]->getOffset());
+        $this->assertEquals(5, $arr[5]->getCount());
     }
 }
\ No newline at end of file