Removed the tokens array.

19 files changed, 31 insertions, 54 deletions

diff --git a/src/Context.php b/src/Context.php
index 1a06baf..9d6927a 100644
--- a/src/Context.php
+++ b/src/Context.php
@@ -268,12 +268,15 @@ abstract class Context
      * @var array
      */
     public static $DATA_TYPES = array(
-        'ARRAY' => 1, 'BIGINT' => 1, 'BINARY VARYING' => 1, 'BINARY' => 1,
-        'BOOLEAN' => 1, 'CHARACTER' => 1, 'CHARACTER' => 1, 'DATE' => 1,
-        'DECIMAL' => 1, 'DOUBLE' => 1, 'FLOAT' => 1, 'FLOAT' => 1, 'INT' => 1,
-        'INTEGER' => 1, 'INTERVAL' => 1, 'MULTISET' => 1, 'NUMERIC' => 1,
-        'REAL' => 1, 'SMALLINT' => 1, 'TIME' => 1, 'TIMESTAMP' => 1,
-        'VARBINARY' => 1, 'VARCHAR' => 1, 'XML' => 1,
+        'ARRAY' => 1, 'BIGINT' => 1, 'BINARY' => 1, 'BINARY VARYING' => 1,
+        'BLOB' => 1, 'BOOLEAN' => 1, 'CHAR' => 1, 'CHARACTER' => 1, 'DATE' => 1,
+        'DATETIME' => 1, 'DECIMAL' => 1, 'DOUBLE' => 1, 'ENUM' => 1,
+        'FLOAT' => 1, 'INT' => 1, 'INTEGER' => 1, 'INTERVAL' => 1,
+        'LONGBLOB' => 1, 'LONGTEXT' => 1, 'MEDIUMBLOB' => 1, 'MEDIUMINT' => 1,
+        'MEDIUMTEXT' => 1, 'MULTISET' => 1, 'NUMERIC' => 1, 'REAL' => 1,
+        'SET' => 1, 'SMALLINT' => 1, 'TEXT' => 1, 'TIME' => 1, 'TIMESTAMP' => 1,
+        'TINYBLOB' => 1, 'TINYINT' => 1, 'TINYTEXT' => 1, 'VARBINARY' => 1,
+        'VARCHAR' => 1, 'XML' => 1, 'YEAR' => 1
     );
 
     // -------------------------------------------------------------------------
diff --git a/src/Fragment.php b/src/Fragment.php
index 86f9d36..63af280 100644
--- a/src/Fragment.php
+++ b/src/Fragment.php
@@ -10,14 +10,6 @@ abstract class Fragment
 {
 
     /**
-     * Array which contains all tokens used to popoluate data inside this
-     * fragment.
-     *
-     * @var array
-     */
-    public $tokens = array();
-
-    /**
      * Parses the tokens given by the lexer in the context of the given parser.
      *
      * @param Parser $parser The parser that serves as context.
diff --git a/src/Fragments/ArrayFragment.php b/src/Fragments/ArrayFragment.php
index 3b796b7..6436823 100644
--- a/src/Fragments/ArrayFragment.php
+++ b/src/Fragments/ArrayFragment.php
@@ -15,13 +15,20 @@ class ArrayFragment extends Fragment
 {
 
     /**
-     * The array.
+     * The array that contains the processed value of each token.
      *
      * @var array
      */
     public $array = array();
 
     /**
+     * The array that contains the unprocessed value of each token.
+     *
+     * @var array
+     */
+    public $raw = array();
+
+    /**
      * @param Parser $parser The parser that serves as context.
      * @param TokensList $list The list of tokens that are being parsed.
      * @param array $options Parameters for parsing.
@@ -74,7 +81,7 @@ class ArrayFragment extends Fragment
                     break;
                 }
                 $ret->array[] = $token->value;
-                $ret->tokens[] = $token;
+                $ret->raw[] = $token->token;
                 $state = 2;
             } elseif ($state === 2) {
                 if (($token->type !== Token::TYPE_OPERATOR) || (($token->value !== ',') && ($token->value !== ')'))) {
diff --git a/src/Fragments/CallKeyword.php b/src/Fragments/CallKeyword.php
index 58ee89b..6147cf4 100644
--- a/src/Fragments/CallKeyword.php
+++ b/src/Fragments/CallKeyword.php
@@ -68,13 +68,11 @@ class CallKeyword extends Fragment
 
             if ($state === 0) {
                 $ret->name = $token->value;
-                $ret->tokens[] = $token;
                 $state = 1;
             } elseif ($state === 1) {
                 if (($token->type === Token::TYPE_OPERATOR) && ($token->value === '(')) {
                     $parameters = ArrayFragment::parse($parser, $list);
                     $ret->parameters = $parameters->array;
-                    $ret->tokens = array_merge($ret->tokens, $parameters->tokens);
                 }
                 break;
             }
diff --git a/src/Fragments/CreateDefFragment.php b/src/Fragments/CreateDefFragment.php
index fefb43d..675ad81 100644
--- a/src/Fragments/CreateDefFragment.php
+++ b/src/Fragments/CreateDefFragment.php
@@ -98,7 +98,6 @@ class CreateDefFragment extends Fragment
                 break;
             }
 
-            $ret->tokens[] = $token;
             $ret->name .= $token->value;
         }
 
diff --git a/src/Fragments/DataTypeFragment.php b/src/Fragments/DataTypeFragment.php
index ad98d0a..ca7cfaf 100644
--- a/src/Fragments/DataTypeFragment.php
+++ b/src/Fragments/DataTypeFragment.php
@@ -80,7 +80,6 @@ class DataTypeFragment extends Fragment
 
             if ($state === 0) {
                 $ret->name = $token->value;
-                $ret->tokens[] = $token;
                 if (!isset(Context::$DATA_TYPES[$token->value])) {
                     $parser->error('Unrecognized data type.', $token);
                 }
@@ -88,11 +87,9 @@ class DataTypeFragment extends Fragment
             } elseif ($state === 1) {
                 if (($token->type === Token::TYPE_OPERATOR) && ($token->value === '(')) {
                     $size = ArrayFragment::parse($parser, $list);
-                    foreach ($size->tokens as $token) {
-                        $ret->size[] = $token->token;
-                    }
-                    $ret->tokens = array_merge($ret->tokens, $size->tokens);
                     ++$list->idx;
+                    $ret->size = (($ret->name === 'ENUM') || ($ret->name === 'SET')) ?
+                        $size->raw : $size->array;
                 }
                 $ret->options = OptionsFragment::parse($parser, $list, static::$OPTIONS);
                 ++$list->idx;
diff --git a/src/Fragments/FieldDefFragment.php b/src/Fragments/FieldDefFragment.php
index 224fefa..3079681 100644
--- a/src/Fragments/FieldDefFragment.php
+++ b/src/Fragments/FieldDefFragment.php
@@ -166,11 +166,10 @@ class FieldDefFragment extends Fragment
                 }
             }
 
-            $expr->tokens[] = $token;
         }
 
         // Last iteration was not saved.
-        if (!empty($expr->tokens)) {
+        if (!empty($expr->name)) {
             $ret[] = $expr;
         }
 
diff --git a/src/Fragments/FieldFragment.php b/src/Fragments/FieldFragment.php
index ccee9d4..9a69a2e 100644
--- a/src/Fragments/FieldFragment.php
+++ b/src/Fragments/FieldFragment.php
@@ -85,7 +85,6 @@ class FieldFragment extends Fragment
             if (($token->type === Token::TYPE_WHITESPACE) || ($token->type === Token::TYPE_COMMENT)) {
                 if (($isExpr) && (!$alias)) {
                     $ret->expr .= $token->token;
-                    $ret->tokens[] = $token;
                 }
                 if (($alias === 0) && (!$isExpr) && (!$period) && (!empty($ret->expr))) {
                     $alias = 1;
@@ -156,7 +155,6 @@ class FieldFragment extends Fragment
                 }
 
                 $ret->expr .= $token->token;
-                $ret->tokens[] = $token;
             }
         }
 
@@ -164,7 +162,7 @@ class FieldFragment extends Fragment
             $parser->error('Alias was expected.', $token);
         }
 
-        if (empty($ret->tokens)) {
+        if (empty($ret->expr)) {
             return null;
         }
 
diff --git a/src/Fragments/FromKeyword.php b/src/Fragments/FromKeyword.php
index fd8c797..c7dc624 100644
--- a/src/Fragments/FromKeyword.php
+++ b/src/Fragments/FromKeyword.php
@@ -58,7 +58,7 @@ class FromKeyword extends Fragment
         }
 
         // Last iteration was not saved.
-        if (!empty($expr->tokens)) {
+        if ($expr !== null) {
             $ret[] = $expr;
         }
 
diff --git a/src/Fragments/IntoKeyword.php b/src/Fragments/IntoKeyword.php
index 1476f44..d56293a 100644
--- a/src/Fragments/IntoKeyword.php
+++ b/src/Fragments/IntoKeyword.php
@@ -92,7 +92,6 @@ class IntoKeyword extends Fragment
                 break;
             }
 
-            $ret->tokens[] = $token;
             if ($state === 0) {
                 $ret->table .= $token->value;
             } elseif ($state === 1) {
diff --git a/src/Fragments/LimitKeyword.php b/src/Fragments/LimitKeyword.php
index 717f554..3e6f592 100644
--- a/src/Fragments/LimitKeyword.php
+++ b/src/Fragments/LimitKeyword.php
@@ -75,9 +75,6 @@ class LimitKeyword extends Fragment
             } else {
                 $ret->row_count = $token->value;
             }
-
-            $ret->tokens[] = $token;
-
         }
 
         if ($offset) {
diff --git a/src/Fragments/OptionsFragment.php b/src/Fragments/OptionsFragment.php
index 7b497fb..3317f10 100644
--- a/src/Fragments/OptionsFragment.php
+++ b/src/Fragments/OptionsFragment.php
@@ -104,8 +104,6 @@ class OptionsFragment extends Fragment
                 $ret->options[$lastOptionId] = $token->value;
                 $lastOption = null;
             }
-            $ret->tokens[] = $token;
-
         }
 
         ksort($ret->options);
diff --git a/src/Fragments/OrderKeyword.php b/src/Fragments/OrderKeyword.php
index 234d10d..4cebc1c 100644
--- a/src/Fragments/OrderKeyword.php
+++ b/src/Fragments/OrderKeyword.php
@@ -59,7 +59,6 @@ class OrderKeyword extends Fragment
                 // Type of ordering. By default, it is `ASC`.
                 if (($token->value === 'ASC') || ($token->value === 'DESC')) {
                     $expr->type = $token->value;
-                    $expr->tokens[] = $token;
                     continue;
                 }
 
@@ -74,13 +73,12 @@ class OrderKeyword extends Fragment
                 continue;
             }
 
-            $expr->tokens[] = $token;
             $expr->column .= $token->token;
 
         }
 
         // Last iteration was not processed.
-        if (!empty($expr->tokens)) {
+        if (!empty($expr->column)) {
             $ret[] = $expr;
         }
 
diff --git a/src/Fragments/ParamDefFragment.php b/src/Fragments/ParamDefFragment.php
index c2582ef..132898f 100644
--- a/src/Fragments/ParamDefFragment.php
+++ b/src/Fragments/ParamDefFragment.php
@@ -112,12 +112,10 @@ class ParamDefFragment extends Fragment
                     break;
                 }
             }
-
-            $expr->tokens[] = $token;
         }
 
         // Last iteration was not saved.
-        if (!empty($expr->tokens)) {
+        if (!empty($expr->name)) {
             $ret[] = $expr;
         }
 
diff --git a/src/Fragments/RenameKeyword.php b/src/Fragments/RenameKeyword.php
index 2aa7df4..c38dae6 100644
--- a/src/Fragments/RenameKeyword.php
+++ b/src/Fragments/RenameKeyword.php
@@ -95,7 +95,6 @@ class RenameKeyword extends Fragment
                 break;
             }
 
-            $expr->tokens[] = $token;
             if ($state == 0) {
                 $expr->old = $token->value;
                 $state = 1;
@@ -107,7 +106,7 @@ class RenameKeyword extends Fragment
         }
 
         // Last iteration was not saved.
-        if (!empty($expr->tokens)) {
+        if (!empty($expr->old)) {
             $ret[] = $expr;
         }
 
diff --git a/src/Fragments/SelectKeyword.php b/src/Fragments/SelectKeyword.php
index 32284df..8848815 100644
--- a/src/Fragments/SelectKeyword.php
+++ b/src/Fragments/SelectKeyword.php
@@ -58,7 +58,7 @@ class SelectKeyword extends Fragment
         }
 
         // Last iteration was not processed.
-        if (!empty($expr->tokens)) {
+        if ($expr !== null) {
             $ret[] = $expr;
         }
 
diff --git a/src/Fragments/SetKeyword.php b/src/Fragments/SetKeyword.php
index 65cd234..284173e 100644
--- a/src/Fragments/SetKeyword.php
+++ b/src/Fragments/SetKeyword.php
@@ -85,17 +85,15 @@ class SetKeyword extends Fragment
                 }
             }
 
-            $expr->tokens[] = $token;
             if ($state === 0) {
                 $expr->column .= $token->value;
             } else { // } else if ($state === 1) {
                 $expr->value = $token->value;
             }
-
         }
 
         // Last iteration was not saved.
-        if (!empty($expr->tokens)) {
+        if (!empty($expr->column)) {
             $ret[] = $expr;
         }
 
diff --git a/src/Fragments/ValuesKeyword.php b/src/Fragments/ValuesKeyword.php
index 7897569..db8c5fd 100644
--- a/src/Fragments/ValuesKeyword.php
+++ b/src/Fragments/ValuesKeyword.php
@@ -96,7 +96,6 @@ class ValuesKeyword extends Fragment
                 break;
             }
 
-            $expr->tokens[] = $token;
             if ($state === 1) {
                 $value .= $token->value;
                 $state = 2;
@@ -105,7 +104,7 @@ class ValuesKeyword extends Fragment
         }
 
         // Last iteration was not saved.
-        if (!empty($expr->tokens)) {
+        if (!empty($expr->values)) {
             $ret[] = $expr;
         }
 
diff --git a/src/Fragments/WhereKeyword.php b/src/Fragments/WhereKeyword.php
index f9466b3..c9ce460 100644
--- a/src/Fragments/WhereKeyword.php
+++ b/src/Fragments/WhereKeyword.php
@@ -64,14 +64,13 @@ class WhereKeyword extends Fragment
 
             // Conditions are delimited by logical operators.
             if (in_array($token->value, static::$OPERATORS, true)) {
-                if (!empty($expr->tokens)) {
+                if (!empty($expr->condition)) {
                     $ret[] = $expr;
                 }
 
                 $expr = new WhereKeyword();
                 $expr->isOperator = true;
                 $expr->condition = $token->value;
-                $epxr->tokens[] = $token;
                 $ret[] = $expr;
 
                 $expr = new WhereKeyword();
@@ -84,13 +83,12 @@ class WhereKeyword extends Fragment
                 break;
             }
 
-            $expr->tokens[] = $token;
             $expr->condition .= $token->token;
 
         }
 
         // Last iteration was not processed.
-        if (!empty($expr->tokens)) {
+        if (!empty($expr->condition)) {
             $ret[] = $expr;
         }