release: version 0.4.0 🚀

.gitea/conventional_commits/generate-version.sh

@@ -46,7 +46,7 @@ create_file() {
 }
 
 get_commit_range() {
-    rm $TEMP_FILE_PATH/messages.txt
+    rm -f $TEMP_FILE_PATH/messages.txt
     if [[ $LAST_TAG =~ $PATTERN ]]; then
         create_file true
     else
@@ -66,17 +66,14 @@ start() {
         echo $message
         if echo $message | grep -Pq '(feat|style)(\([\w]+\))?!:([a-zA-Z0-9-_!\&\.\%\(\)\=\w\s]+)\s?(,?\s?)((ref(s?):?\s?)(([A-Z0-9]+\-[0-9]+)|(NOISSUE)))'; then
             increment_type="major"
-            echo "a"
             break
         elif echo $message | grep -Pq '(feat|style)(\([\w]+\))?:([a-zA-Z0-9-_!\&\.\%\(\)\=\w\s]+)\s?(,?\s?)((ref(s?):?\s?)(([A-Z0-9]+\-[0-9]+)|(NOISSUE)))'; then
             if [ -z "$increment_type" ] || [ "$increment_type" == "patch" ]; then
                 increment_type="minor"
-                echo "b"
             fi 
         elif echo $message | grep -Pq '(build|fix|perf|refactor|revert)(\(.+\))?:\s([a-zA-Z0-9-_!\&\.\%\(\)\=\w\s]+)\s?(,?\s?)((ref(s?):?\s?)(([A-Z0-9]+\-[0-9]+)|(NOISSUE)))'; then
             if [ -z "$increment_type" ]; then
                 increment_type="patch"
-                echo "c"
             fi
         fi
     done < $TEMP_FILE_PATH/messages.txt
@@ -86,8 +83,9 @@ start() {
         echo "New version: $new_version"
 
         gitchangelog | grep -v "[rR]elease:" > HISTORY.md
-        git add DotnetTestLib/VERSION HISTORY.md
-        echo $new_version > DotnetTestLib/VERSION
+        echo $new_version > TextParser/VERSION
+        git add TextParser/VERSION
+        git add HISTORY.md
         git commit -m "release: version $new_version 🚀"
         echo "creating git tag : $new_version"
         git tag $new_version

.gitea/workflows/release.yml

@@ -64,5 +64,5 @@ jobs:
           QUOTED_REPOSITORY_SOURCE_NAME=$(dotnet nuget config get all | grep "/packages/${REPOSITORY_OWNER}/nuget/index.json" | awk '{print $2}' | awk -F= '{print $2}')
           REPOSITORY_SOURCE_NAME=${QUOTED_REPOSITORY_SOURCE_NAME:1:-1}
         fi
-        dotnet pack --include-symbols --include-source -p:PackageVersion=$(cat $REPOSITORY_NAME/VERSION) DotnetTestLib.sln
+        dotnet pack --include-symbols --include-source -p:PackageVersion=$(cat $REPOSITORY_NAME/VERSION) TextParser.sln
         dotnet nuget push -k $GITEA_PAT --source $REPOSITORY_SOURCE_NAME $REPOSITORY_NAME/bin/Release/$REPOSITORY_NAME.$(cat $REPOSITORY_NAME/VERSION).symbols.nupkg

HISTORY.md

@@ -4,8 +4,66 @@ Changelog
 
 (unreleased)
 ------------
-- More bugfixes. [Simon Diesenreiter]
-- Fix linting errors. [Simon Diesenreiter]
-- Initial commit. [Simon Diesenreiter]
+- Style: fix linting errors, ref: NOISSUE. [Simon Diesenreiter]
+- Feat: implement greedy repetition, ref: A24-13. [Simon Diesenreiter]
+
+
+0.3.0 (2024-12-02)
+------------------
+- Feat: ci pipeline fix for releases, ref NOISSUE. [Simon Diesenreiter]
+
+
+0.2.3 (2024-12-02)
+------------------
+
+Fix
+~~~
+- More release script fixes,ref: NOISSUE. [Simon Diesenreiter]
+
+Other
+~~~~~
+
+
+0.2.2 (2024-12-02)
+------------------
+
+Fix
+~~~
+- Increment version,refs:NOISSUE. [Simon Diesenreiter]
+
+Other
+~~~~~
+
+
+0.2.1 (2024-12-02)
+------------------
+
+Fix
+~~~
+- Fix file inclusion in tag commits, ref: NOISSUE. [Simon Diesenreiter]
+
+
+0.2.0 (2024-12-02)
+------------------
+
+Fix
+~~~
+- Fix some bugs and tests, ref: A24-3. [Simon Diesenreiter]
+
+Other
+~~~~~
+- Ci: fix release shortcut, ref: NOISSUE. [Simon Diesenreiter]
+- Feat: add text parser and output format options, ref: A24-3. [Simon
+  Diesenreiter]
+
+
+0.1.0 (2024-12-01)
+------------------
+- Feat: added initial implementation of TextParser, ref: A24-3. [Simon
+  Diesenreiter]
+- ✅ Ready to clone and code. [simon]
+- Ci: initial commit for triggering migration, ref: NOISSUE. [Simon
+  Diesenreiter]
+- Initial commit. [Projects <>]
 
 

Makefile

@@ -2,7 +2,7 @@
 
 .PHONY: issetup
 issetup:
-	@[ -f .git/hooks/commit-msg ] || [ -v SKIP_MAKE_SETUP_CHECK ] || (echo "You must run 'make setup' first to initialize the repo!" && exit 1)
+	@[ -f .git/hooks/commit-msg ] || [ -n SKIP_MAKE_SETUP_CHECK ] || (echo "You must run 'make setup' first to initialize the repo!" && exit 1)
 
 .PHONY: setup
 setup:
@@ -45,15 +45,7 @@ clean: issetup           ## Clean unused files.
 
 .PHONY: release
 release: issetup         ## Create a new tag for release.
-	@echo "WARNING: This operation will create a version tag and push to gitea"
-	@read -p "Version? (provide the next x.y.z semver) : " TAG; echo "$$TAG" > TextParser/VERSION
-	@gitchangelog > HISTORY.md
-	@git add TextParser/VERSION HISTORY.md
-	@git commit -m "release: version $$(cat TextParser/VERSION) 🚀"
-	@echo "creating git tag : $$(cat TextParser/VERSION)"
-	@git tag $$(cat TextParser/VERSION)
-	@git push -u origin HEAD --tags
-	@echo "Gitea Actions will detect the new tag and release the new version."
+	@./.gitea/conventional_commits/generate-version.sh
 
 .PHONY: docs
 docs: issetup            ## Build the documentation.

TextParser.Tests/TextParserTests.cs

@@ -9,9 +9,15 @@ public class TextParserTests
 {
     private const string testInput1 = "2 4 6 8";
     private const string testInput2 = "2 ab ba 8 cd dc";
+    private const string testInput3 = @"2 4 6 1
+                                        3 5 7 2
+                                        4 6 8 3";
+    private const string testInput4 = @"2 ab ba fd er sd 
+                                        8 cd dc 
+                                        7 uh 6 yp rt";
 
     [Fact]
-    public void TestSimpleRepetition()
+    public void LineParser_TestSimpleRepetition()
     {
         var schemaBuilder = new InputSchemaBuilder();
         var schema = schemaBuilder
@@ -20,7 +26,7 @@ public class TextParserTests
                         .EndRepetition()
                         .Build();
 
-        var parser = new TextParser(schema);
+        var parser = new LineParser(schema);
         var tokens = parser.ParseLine(testInput1);
 
         Assert.Equal(4, tokens.Count);
@@ -35,7 +41,7 @@ public class TextParserTests
     }
 
     [Fact]
-    public void TestSimpleInput()
+    public void LineParser_TestSimpleInput()
     {
         var schemaBuilder = new InputSchemaBuilder();
         var schema = schemaBuilder
@@ -45,7 +51,7 @@ public class TextParserTests
                         .Expect(InputType.Integer)
                         .Build();
 
-        var parser = new TextParser(schema);
+        var parser = new LineParser(schema);
         var tokens = parser.ParseLine(testInput1);
 
         Assert.Equal(4, tokens.Count);
@@ -57,11 +63,11 @@ public class TextParserTests
         Assert.Equal(4, (tokens[1] as IntegerToken)?.GetValue());
         Assert.Equal(6, (tokens[2] as IntegerToken)?.GetValue());
         Assert.Equal(8, (tokens[3] as IntegerToken)?.GetValue());
-    }
-
-
+    }
+
+
     [Fact]
-    public void TestNestedRepetition()
+    public void LineParser_TestNestedRepetition()
     {
         var schemaBuilder = new InputSchemaBuilder();
         var schema = schemaBuilder
@@ -73,7 +79,7 @@ public class TextParserTests
                         .EndRepetition()
                         .Build();
 
-        var parser = new TextParser(schema);
+        var parser = new LineParser(schema);
         var tokens = parser.ParseLine(testInput2);
 
         Assert.Equal(6, tokens.Count);
@@ -90,4 +96,107 @@ public class TextParserTests
         Assert.Equal("cd", (tokens[4] as StringToken)?.GetValue());
         Assert.Equal("dc", (tokens[5] as StringToken)?.GetValue());
     }
+
+    [Fact]
+    public void TextParser_TestRepetitionAsRows()
+    {
+        var schemaBuilder = new InputSchemaBuilder();
+        var schema = schemaBuilder
+                        .Repeat(4)
+                            .Expect(InputType.Integer)
+                        .EndRepetition()
+                        .Build();
+
+        var parser = new TextParser(schema);
+        var rows = parser
+                        .SetInputText(testInput3)
+                        .Parse()
+                        .AsRows<int>();
+
+        Assert.Equal(3, rows.Count);
+        Assert.Equal(4, rows[0].Length);
+        Assert.Equal(2, rows[0][0]);
+        Assert.Equal(4, rows[0][1]);
+        Assert.Equal(6, rows[0][2]);
+        Assert.Equal(1, rows[0][3]);
+        Assert.Equal(3, rows[1][0]);
+        Assert.Equal(5, rows[1][1]);
+        Assert.Equal(7, rows[1][2]);
+        Assert.Equal(2, rows[1][3]);
+        Assert.Equal(4, rows[2][0]);
+        Assert.Equal(6, rows[2][1]);
+        Assert.Equal(8, rows[2][2]);
+        Assert.Equal(3, rows[2][3]);
+    }
+
+    [Fact]
+    public void TextParser_TestRepetitionAsColumns()
+    {
+        var schemaBuilder = new InputSchemaBuilder();
+        var schema = schemaBuilder
+                        .Repeat(4)
+                            .Expect(InputType.Integer)
+                        .EndRepetition()
+                        .Build();
+
+        var parser = new TextParser(schema);
+        var columns = parser
+                        .SetInputText(testInput3)
+                        .Parse()
+                        .AsColumns<int>();
+
+        Assert.Equal(4, columns.Count);
+        Assert.Equal(3, columns[0].Length);
+        Assert.Equal(2, columns[0][0]);
+        Assert.Equal(3, columns[0][1]);
+        Assert.Equal(4, columns[0][2]);
+        Assert.Equal(4, columns[1][0]);
+        Assert.Equal(5, columns[1][1]);
+        Assert.Equal(6, columns[1][2]);
+        Assert.Equal(6, columns[2][0]);
+        Assert.Equal(7, columns[2][1]);
+        Assert.Equal(8, columns[2][2]);
+        Assert.Equal(1, columns[3][0]);
+        Assert.Equal(2, columns[3][1]);
+        Assert.Equal(3, columns[3][2]);
+    }
+
+    [Fact]
+    public void TextParser_TestGreedyRepetitionAsRows()
+    {
+        var schemaBuilder = new InputSchemaBuilder();
+        var schema = schemaBuilder
+                        .Repeat()
+                            .Expect(InputType.Integer)
+                            .Repeat()
+                                .Expect(InputType.String)
+                            .EndRepetition()
+                        .EndRepetition()
+                        .Build();
+
+        var parser = new TextParser(schema);
+        var rows = parser
+                        .SetInputText(testInput4)
+                        .Parse()
+                        .AsRawData();
+
+        Assert.Equal(3, rows.Count);
+        Assert.Equal(6, rows[0].Count);
+        Assert.Equal(3, rows[1].Count);
+        Assert.Equal(5, rows[2].Count);
+        Assert.Equal(InputType.Integer, rows[0][0].GetInputType());
+        Assert.Equal(InputType.String, rows[0][1].GetInputType());
+        Assert.Equal(InputType.String, rows[0][2].GetInputType());
+        Assert.Equal(InputType.String, rows[0][3].GetInputType());
+        Assert.Equal(InputType.String, rows[0][4].GetInputType());
+        Assert.Equal(InputType.String, rows[0][5].GetInputType());
+        Assert.Equal(InputType.Integer, rows[1][0].GetInputType());
+        Assert.Equal(InputType.String, rows[1][1].GetInputType());
+        Assert.Equal(InputType.String, rows[1][2].GetInputType());
+        Assert.Equal(InputType.Integer, rows[2][0].GetInputType());
+        Assert.Equal(InputType.String, rows[2][1].GetInputType());
+        Assert.Equal(InputType.Integer, rows[2][2].GetInputType());
+        Assert.Equal(InputType.String, rows[2][3].GetInputType());
+        Assert.Equal(InputType.String, rows[2][4].GetInputType());
+    }
 }

TextParser/LineParser.cs

@@ -0,0 +1,37 @@
+namespace Parsing;
+
+using Parsing.Schema;
+using Parsing.Tokenization;
+
+public class LineParser
+{
+    private string[] delimiters;
+    private bool removeEmptyEntries = false;
+    private InputSchema schema;
+    private InputSchemaContext context;
+
+    public LineParser(InputSchema schema, string[]? delimiters = null, bool removeEmptyEntries = true)
+    {
+        this.delimiters = delimiters ?? new string[] { " " };
+        this.removeEmptyEntries = removeEmptyEntries;
+        this.schema = schema;
+        this.context = this.schema.CreateContext();
+    }
+
+    private string[] ParseLineIntoWords(string line)
+    {
+        var options = StringSplitOptions.TrimEntries;
+        if (this.removeEmptyEntries)
+        {
+            options = options | StringSplitOptions.RemoveEmptyEntries;
+        }
+        return line.Split(this.delimiters, options);
+    }
+
+    public List<IToken> ParseLine(string line)
+    {
+        this.context = this.schema.CreateContext();
+        var words = this.ParseLineIntoWords(line);
+        return this.schema.ProcessWordList(words);
+    }
+}

TextParser/Schema/BuildingBlocks/BuildingBlockBase.cs

@@ -12,6 +12,8 @@ abstract class BuildingBlockBase : IBuildingBlock
 
     public abstract bool CanParseWord(InputProvider inputs);
 
+    public abstract bool CanParseWord(string word);
+
     public abstract BlockType GetBlockType();
 
     public virtual bool IsRepetitionType()
@@ -19,7 +21,7 @@ abstract class BuildingBlockBase : IBuildingBlock
         return false;
     }
 
-    public virtual bool CheckIsDoneParsingAndReset()
+    public virtual bool CheckIsDoneParsingAndReset(InputProvider inputs)
     {
         // most blocks are always done parsing after consuming a token
         // repetition blocks can consume multiple tokens

TextParser/Schema/BuildingBlocks/FixedRepetitionBlock.cs

@@ -48,6 +48,21 @@ class FixedRepetitionBlock : BuildingBlockBase
         return result;
     }
 
+    public override bool CanParseWord(string word)
+    {
+        bool result;
+        if (this.repetitionCount == 0)
+        {
+            result = false;
+        }
+        else
+        {
+            result = inputSchema.CanProcessNextWord(context, word);
+        }
+
+        return result;
+    }
+
     public override BlockType GetBlockType()
     {
         return BlockType.FixedRepetition;
@@ -58,7 +73,7 @@ class FixedRepetitionBlock : BuildingBlockBase
         return true;
     }
 
-    public override bool CheckIsDoneParsingAndReset()
+    public override bool CheckIsDoneParsingAndReset(InputProvider inputs)
     {
         // we are done parsing once all repetitions are exhausted
         var result = this.repetitionCount == 0;

TextParser/Schema/BuildingBlocks/GreedyRepetitionBlock.cs

@@ -0,0 +1,57 @@
+namespace Parsing.Schema.BuildingBlocks;
+
+using System.IO.Pipelines;
+using Parsing.Tokenization;
+
+class GreedyRepetitionBlock : BuildingBlockBase
+{
+    private InputSchema inputSchema;
+    private InputSchemaContext context;
+
+    public GreedyRepetitionBlock(InputSchema inputSchema)
+    {
+        this.inputSchema = inputSchema;
+        this.context = this.inputSchema.CreateContext();
+    }
+
+    public override IToken ParseWord(InputProvider inputs)
+    {
+        var result = inputSchema.ProcessNextWord(context, inputs);
+        if (!this.CanParseWord(inputs))
+        {
+            this.context = this.inputSchema.CreateContext();
+        }
+        return result;
+    }
+
+    public override bool CanParseWord(InputProvider inputs)
+    {
+        return inputSchema.CanProcessNextWord(context, inputs) && inputs.CanYieldWord();
+    }
+
+    public override bool CanParseWord(string word)
+    {
+        return inputSchema.CanProcessNextWord(context, word);
+    }
+
+    public override BlockType GetBlockType()
+    {
+        return BlockType.GreedyRepetition;
+    }
+
+    public override bool IsRepetitionType()
+    {
+        return true;
+    }
+
+    public override bool CheckIsDoneParsingAndReset(InputProvider inputs)
+    {
+        // we are done parsing greedily once the next token doesn't match anymore
+        var result = !this.CanParseWord(inputs);
+        if (result)
+        {
+            this.context = this.inputSchema.CreateContext();
+        }
+        return result;
+    }
+}

TextParser/Schema/BuildingBlocks/IBuildingBlock.cs

@@ -8,9 +8,11 @@ public interface IBuildingBlock
 
     public bool CanParseWord(InputProvider inputs);
 
+    public bool CanParseWord(string word);
+
     public BlockType GetBlockType();
 
     public bool IsRepetitionType();
 
-    public bool CheckIsDoneParsingAndReset();
+    public bool CheckIsDoneParsingAndReset(InputProvider inputs);
 }

TextParser/Schema/BuildingBlocks/IntegerBlock.cs

@@ -18,12 +18,16 @@ class IntegerBlock : BuildingBlockBase
     {
         using (inputs.GetLookaheadContext())
         {
-            int number = 0;
-            var success = int.TryParse(inputs.YieldWord(), out number);
-            return success;
+            return this.CanParseWord(inputs.YieldWord());
         }
     }
 
+    public override bool CanParseWord(string word)
+    {
+        int number = 0;
+        return int.TryParse(word, out number);
+    }
+
     public override BlockType GetBlockType()
     {
         return BlockType.Integer;

TextParser/Schema/BuildingBlocks/StringBlock.cs

@@ -16,6 +16,24 @@ class StringBlock : BuildingBlockBase
 
     public override bool CanParseWord(InputProvider inputs)
     {
+        string word = string.Empty;
+        using (inputs.GetLookaheadContext())
+        {
+            word = inputs.YieldWord();
+        }
+
+        return this.CanParseWord(word);
+    }
+
+    public override bool CanParseWord(string word)
+    {
+        // Here we need to ensure we are not matching any non-string tokens, since string can match pretty much anything
+        IntegerBlock intBlock = new IntegerBlock();
+        if (intBlock.CanParseWord(word))
+        {
+            return false;
+        }
+
         return true;
     }
 

TextParser/Schema/InputSchema.cs

@@ -29,7 +29,7 @@ public class InputSchema
     {
         var nextBlock = this.buildingBlocks[currentContext.lastProcessedBlockIndex];
         var token = nextBlock.ParseWord(inputs);
-        if (!nextBlock.IsRepetitionType() || nextBlock.CheckIsDoneParsingAndReset())
+        if (!nextBlock.IsRepetitionType() || nextBlock.CheckIsDoneParsingAndReset(inputs))
         {
             currentContext.lastProcessedBlockIndex++;
             currentContext.HasFinished = currentContext.lastProcessedBlockIndex >= this.buildingBlocks.Count;
@@ -50,7 +50,17 @@ public class InputSchema
         }
     }
 
-    public IList<IToken> ProcessWordList(string[] words)
+    public bool CanProcessNextWord(InputSchemaContext currentContext, string word)
+    {
+        if (currentContext.HasFinished)
+        {
+            return false;
+        }
+        var nextBlock = this.buildingBlocks[currentContext.lastProcessedBlockIndex];
+        return nextBlock.CanParseWord(word);
+    }
+
+    public List<IToken> ProcessWordList(string[] words)
     {
         List<IToken> tokens = new List<IToken>();
         InputProvider inputs = new InputProvider(words);

TextParser/Schema/InputSchemaBuilder.cs

@@ -38,6 +38,15 @@ public class InputSchemaBuilder
         return newInputSchemaBuilder;
     }
 
+    public InputSchemaBuilder Repeat()
+    {
+        // add another layer of parsing
+        var newInputSchemaBuilder = new RepetitionSchemaBuilder(this);
+        newInputSchemaBuilder.RepetitionType = RepetitionType.GreedyRepetition;
+
+        return newInputSchemaBuilder;
+    }
+
     public InputSchemaBuilder EndRepetition()
     {
         // return back to upper layer of parsing
@@ -54,6 +63,9 @@ public class InputSchemaBuilder
             case RepetitionType.FixedRepetition:
                 oldInputSchemaBuilder.schema.AddBuildingBlock(new FixedRepetitionBlock(currentSchema, currentBuilder.NumRepetition));
                 break;
+            case RepetitionType.GreedyRepetition:
+                oldInputSchemaBuilder.schema.AddBuildingBlock(new GreedyRepetitionBlock(currentSchema));
+                break;
             default:
                 throw new Exception("Unrecognized RepetitionType");
         }

TextParser/TextParser.cs

@@ -1,36 +1,40 @@
 namespace Parsing;
 
+using System;
+using System.Collections.Generic;
 using Parsing.Schema;
 using Parsing.Tokenization;
 
-public class TextParser
+public class TextParser : TokenConverter
 {
-    private string[] delimiters;
-    private bool removeEmptyEntries = false;
-    private InputSchema schema;
-    private InputSchemaContext context;
+    private LineParser lineParser;
+    private string[] lines;
+    private bool removeEmptyEntries;
 
-    public TextParser(InputSchema schema, string[]? delimiters = null, bool removeEmptyEntries = true)
+    public TextParser(InputSchema schema, string[]? delimiters = null, bool removeEmptyEntries = true) : base()
     {
-        this.delimiters = delimiters ?? new string[] { " " };
+        this.lineParser = new LineParser(schema, delimiters, removeEmptyEntries);
+        this.lines = new string[] { };
         this.removeEmptyEntries = removeEmptyEntries;
-        this.schema = schema;
-        this.context = this.schema.CreateContext();
     }
 
-    private string[] ParseLineIntoWords(string line)
+    public TextParser SetInputText(string text)
     {
         var options = StringSplitOptions.TrimEntries;
-        if (this.removeEmptyEntries)
+        if (removeEmptyEntries)
         {
             options = options | StringSplitOptions.RemoveEmptyEntries;
         }
-        return line.Split(this.delimiters, options);
+        this.lines = text.Split("\n", options);
+        return this;
     }
 
-    public IList<IToken> ParseLine(string line)
+    public TextParser Parse()
     {
-        var words = this.ParseLineIntoWords(line);
-        return this.schema.ProcessWordList(words);
+        foreach (var line in this.lines)
+        {
+            this.rawTokens.Add(this.lineParser.ParseLine(line));
+        }
+        return this;
     }
 }

TextParser/TokenConverter.cs

@@ -0,0 +1,119 @@
+namespace Parsing;
+
+using System;
+using System.Collections.Generic;
+using Parsing.Schema;
+using Parsing.Tokenization;
+
+public class TokenConverter
+{
+    protected List<List<IToken>> rawTokens = new List<List<IToken>>();
+
+    public TokenConverter()
+    {
+    }
+
+    private List<T> AsGenericCollection<T, U>() where T : ICollection<U>, new()
+    {
+        List<T> returnData = new List<T>();
+        foreach (var tokenRow in this.rawTokens)
+        {
+            T newRow = new T();
+            foreach (IToken token in tokenRow)
+            {
+                if (token == null)
+                {
+                    throw new Exception("No token was provided, but token was expected!");
+                }
+                IValueToken<U>? valueToken = token as IValueToken<U>;
+                if (valueToken == null)
+                {
+                    throw new Exception("Provided token is not a ValueToken");
+                }
+                newRow.Add(valueToken.GetValue());
+            }
+
+            returnData.Add(newRow);
+        }
+        return returnData;
+    }
+
+    private void CheckConversionPrerequisites()
+    {
+        // in order to convert rows to columns or grid we require every row to have the same length
+        int rowLength = this.rawTokens[0].Count;
+
+        foreach (var tokenRow in this.rawTokens)
+        {
+            if (tokenRow.Count != rowLength)
+            {
+                throw new Exception("Attempted to convert token dataset that is not able to be converted!");
+            }
+        }
+    }
+
+    public List<T[]> AsRows<T>()
+    {
+        var listRows = this.AsListRows<T>();
+        var newList = new List<T[]>();
+
+        foreach (var rowList in listRows)
+        {
+            newList.Add(rowList.ToArray());
+        }
+
+        return newList;
+    }
+
+    public List<List<T>> AsListRows<T>()
+    {
+        return this.AsGenericCollection<List<T>, T>();
+    }
+
+    public List<T[]> AsColumns<T>()
+    {
+        var listColumns = this.AsListColumns<T>();
+        var newList = new List<T[]>();
+
+        foreach (var columnList in listColumns)
+        {
+            newList.Add(columnList.ToArray());
+        }
+
+        return newList;
+    }
+
+    public List<List<T>> AsListColumns<T>()
+    {
+        this.CheckConversionPrerequisites();
+        var rows = AsListRows<T>();
+
+        var columns = new List<List<T>>();
+        for (int i = 0; i < rows[0].Count; i++)
+        {
+            columns.Add(new List<T>());
+        }
+
+        foreach (var row in rows)
+        {
+            for (int i = 0; i < row.Count; i++)
+            {
+                columns[i].Add(row[i]);
+            }
+        }
+
+        return columns;
+    }
+
+    public T[][] AsGrid<T>()
+    {
+        this.CheckConversionPrerequisites();
+        var rowsList = AsRows<T>();
+        return rowsList.ToArray();
+    }
+
+    public List<List<IToken>> AsRawData()
+    {
+        return this.rawTokens;
+    }
+}

TextParser/Tokenization/IValueToken.cs

@@ -1,4 +1,6 @@
-namespace Parsing.Tokenization;
+using System;
+
+namespace Parsing.Tokenization;
 
 public interface IValueToken<T> : IToken
 {

TextParser/Tokenization/InputProvider.cs

@@ -37,14 +37,14 @@ public class InputProvider
         return new InputProvider.LookaheadContext(this);
     }
 
+    public bool CanYieldWord()
+    {
+        return this.CurrentPosition < this.words.Length;
+    }
+
     public string YieldWord()
     {
-        Console.WriteLine("current words:");
-        foreach (var word in words)
-        {
-            Console.WriteLine(word);
-        }
-        if (this.CurrentPosition > this.words.Length)
+        if (!this.CanYieldWord())
         {
             return string.Empty;
         }

TextParser/VERSION

@@ -1 +1 @@
-0.1.1
+0.4.0