release: version 0.10.0 🚀

HISTORY.md

@@ -4,6 +4,81 @@ Changelog
 
 (unreleased)
 ------------
+- Feat: add support for custom token types and longs, ref: NOISSUE.
+  [Simon Diesenreiter]
+
+
+0.9.3 (2024-12-13)
+------------------
+
+Fix
+~~~
+- Remove duplicate TokenConverter definition, ref: NOISSUE. [Simon
+  Diesenreiter]
+
+Other
+~~~~~
+
+
+0.9.2 (2024-12-13)
+------------------
+
+Fix
+~~~
+- More bugfixes, ref: NOISSUE. [Simon Diesenreiter]
+
+Other
+~~~~~
+
+
+0.9.1 (2024-12-13)
+------------------
+
+Fix
+~~~
+- Fix build issues, ref: NOISSUE. [Simon Diesenreiter]
+
+Other
+~~~~~
+
+
+0.9.0 (2024-12-13)
+------------------
+- Feat: add filter option to TokenConverter, ref: NOISSUE. [Simon
+  Diesenreiter]
+
+
+0.8.0 (2024-12-12)
+------------------
+- Feat: adding sensible index constructors refs: NOISSUE. [Simon
+  Diesenreiter]
+
+
+0.7.2 (2024-12-05)
+------------------
+
+Fix
+~~~
+- Add some missing API methods, ref: NOISSUE. [Simon Diesenreiter]
+
+Other
+~~~~~
+
+
+0.7.1 (2024-12-05)
+------------------
+
+Fix
+~~~
+- Allow for parsing single chars as input, ref: NOISSUE. [Simon
+  Diesenreiter]
+
+Other
+~~~~~
+
+
+0.7.0 (2024-12-05)
+------------------
 - Feat: implement generic data set manipulator, ref: NOISSUE. [Simon
   Diesenreiter]
 

TextParser.Tests/TextParserTests.cs

@@ -30,6 +30,10 @@ public class TextParserTests
     private const string testInput9 = @"2 4 6 4 1 
                                         3 5 4 7 6 
                                         4 6 8 3 9";
+    private const string testInput10 = @"abc 
+                                         bca 
+                                         cab";
+    private const string testInput11 = @"2 x y 4 x y 6 x y 4 x y 1 x y";
 
     [Fact]
     public void LineParser_TestSimpleRepetition()
@@ -322,7 +326,7 @@ public class TextParserTests
 
     [Fact]
     public void DataManipulator_SimpleOneDimensionalTest()
-    {
+    {
         var schemaBuilder = new InputSchemaBuilder();
         var schema = schemaBuilder
                         .Repeat()
@@ -351,7 +355,7 @@ public class TextParserTests
 
     [Fact]
     public void DataManipulator_SimpleTwoDimensionalTest()
-    {
+    {
         var schemaBuilder = new InputSchemaBuilder();
         var schema = schemaBuilder
                         .Repeat()
@@ -389,4 +393,54 @@ public class TextParserTests
         Assert.Equal(Direction.SE, searchResults[4].Direction);
         Assert.Equal(Direction.W, searchResults[5].Direction);
     }
+
+    [Fact]
+    public void TextParser_TestReadingChars()
+    {
+        var schemaBuilder = new InputSchemaBuilder();
+        var schema = schemaBuilder
+                        .Repeat()
+                            .Expect(InputType.Char)
+                        .EndRepetition()
+                        .Build();
+
+        var parser = new TextParser<InputSchemaContext>(schema);
+        var row = parser
+                        .SetInputText(testInput10)
+                        .Parse()
+                        .AsListRows<string>();
+
+        Assert.Equal(3, row.Count);
+        Assert.Equal("a", row[0][0]);
+        Assert.Equal(3, row[0].Count);
+        Assert.Equal(3, row[1].Count);
+        Assert.Equal(3, row[2].Count);
+    }
+
+    [Fact]
+    public void TextParser_TestFilter()
+    {
+        var schemaBuilder = new InputSchemaBuilder();
+        var schema = schemaBuilder
+                        .Repeat()
+                            .Expect(InputType.Integer)
+                            .Expect(InputType.Char)
+                            .Expect(InputType.Char)
+                        .EndRepetition()
+                        .Build();
+
+        var parser = new TextParser<InputSchemaContext>(schema);
+        var numbers = parser
+                        .SetInputText(testInput11)
+                        .Parse()
+                        .Filter(InputType.Integer)
+                        .AsSingleStream<int>();
+
+        Assert.Equal(5, numbers.Count);
+        Assert.Equal(2, numbers[0]);
+        Assert.Equal(4, numbers[1]);
+        Assert.Equal(6, numbers[2]);
+        Assert.Equal(4, numbers[3]);
+        Assert.Equal(1, numbers[4]);
+    }
 }

TextParser/Data/DataManipulationHelpers.cs

@@ -20,4 +20,24 @@ public static class DataManipulationHelpers
     {
         return reducer(data);
     }
+
+    public static List<TNewType> TransformData<TType, TNewType>(this List<TType> data, Func<TType, TNewType> transformer)
+    {
+        var newList = new List<TNewType>();
+        foreach (TType dataItem in data)
+        {
+            newList.Add(transformer(dataItem));
+        }
+        return newList;
+    }
+
+    public static List<TNewType> TransformData<TType, TNewType>(this List<List<TType>> data, Func<List<TType>, TNewType> transformer)
+    {
+        var newList = new List<TNewType>();
+        foreach (List<TType> dataItemList in data)
+        {
+            newList.Add(transformer(dataItemList));
+        }
+        return newList;
+    }
 }

TextParser/Data/DataSetManipulatorBase.cs

@@ -4,11 +4,22 @@ namespace Parsing.Data;
 
 public class SearchResult<TIndexType>
 {
+    public SearchResult(IDataIndex<TIndexType> dataIndex)
+    {
+        this.DataIndex = dataIndex;
+    }
+
     public IDataIndex<TIndexType>? DataIndex { get; set; }
 }
 
 public class DirectionalSearchResult<TIndexType> : SearchResult<TIndexType>
 {
+    public DirectionalSearchResult(IDataIndex<TIndexType> dataIndex, Direction direction, int length): base(dataIndex)
+    {
+        this.Direction = direction;
+        this.Length = length;
+    }
+
     public Direction Direction { get; set; }
     public int Length { get; set; }
 }
@@ -111,10 +122,7 @@ public abstract class DataSetManipulatorBase<TCollectedType, TDataType, TIndexTy
                 }
                 if (searchIndex == data.Count)
                 {
-                    var result = new DirectionalSearchResult<TIndexType>();
-                    result.DataIndex = currentPosition;
-                    result.Direction = direction;
-                    result.Length = searchIndex;
+                    var result = new DirectionalSearchResult<TIndexType>(currentPosition, direction, searchIndex);
                     results.Add(result);
                 }
             }
@@ -123,7 +131,7 @@ public abstract class DataSetManipulatorBase<TCollectedType, TDataType, TIndexTy
         return results;
     }
 
-    public List<DirectionalSearchResult<TIndexType>> FindInSet(List<TDataType> data)
+    public List<DirectionalSearchResult<TIndexType>> FindInSet(List<TDataType> data, Direction directions)
     {
         var result = new List<DirectionalSearchResult<TIndexType>>();
 
@@ -131,11 +139,16 @@ public abstract class DataSetManipulatorBase<TCollectedType, TDataType, TIndexTy
         var startingPoints = this.FindInSet(data[0]);
         foreach (var startingPoint in startingPoints)
         {
-            foreach (var results in this.FindAtPosition(startingPoint.DataIndex, data))
+            foreach (var results in this.FindAtPosition(startingPoint.DataIndex, data, directions))
             {
                 result.AddRange(results);
             }
         }
         return result;
     }
+
+    public List<DirectionalSearchResult<TIndexType>> FindInSet(List<TDataType> data)
+    {
+        return this.FindInSet(data, this.ValidDirections());
+    }
 }

TextParser/Data/DefaultOneDimensionalManipulator.cs

@@ -49,8 +49,7 @@ public class DefaultOneDimensionalManipulator<TDataType> : DataSetManipulatorBas
         {
             if (EqualityComparer<TDataType>.Default.Equals(this.dataSet[i], data))
             {
-                var singleResult = new SearchResult<int>();
-                singleResult.DataIndex = new DefaultPositionalDataIndex(i);
+                var singleResult = new SearchResult<int>(new DefaultPositionalDataIndex(i));
                 results.Add(singleResult);
             }
         }

TextParser/Data/DefaultTwoDimensionalManipulator.cs

@@ -71,8 +71,7 @@ public class DefaultTwoDimensionalManipulator<TDataType> : DataSetManipulatorBas
             {
                 if (EqualityComparer<TDataType>.Default.Equals(this.dataSet[this.dataSet.Count - y - 1][x], data))
                 {
-                    var singleResult = new SearchResult<int>();
-                    singleResult.DataIndex = new DefaultPositionalDataIndex(x, y);
+                    var singleResult = new SearchResult<int>(new DefaultPositionalDataIndex(x, y));
                     results.Add(singleResult);
                 }
             }

TextParser/Data/TokenConverter.cs

@@ -13,7 +13,7 @@ public class TokenConverter
     {
     }
 
-    private List<T> AsGenericCollection<T, U>() where T : ICollection<U>, new()
+    private List<T> AsGenericCollection<T, U>() where T : List<U>, new()
     {
         List<T> returnData = new List<T>();
         foreach (var tokenRow in this.rawTokens)
@@ -25,11 +25,15 @@ public class TokenConverter
                 {
                     throw new Exception("No token was provided, but token was expected!");
                 }
-                IValueToken<U>? valueToken = token as IValueToken<U>;
-                if (valueToken == null)
+
+                if (!token.GetType().IsAssignableTo(typeof(IValueToken<U>)))
                 {
-                    throw new Exception("Provided token is not a ValueToken");
+                    Console.WriteLine(token.GetText());
+                    Type t = token.GetType();
+                    throw new Exception("Provided token is not a ValueToken - type: " + t.ToString());
                 }
+
+                IValueToken<U> valueToken = token as IValueToken<U>;
                 newRow.Add(valueToken.GetValue());
             }
 
@@ -156,4 +160,26 @@ public class TokenConverter
 
         return newList;
     }
+
+    public TokenConverter Filter(params InputType[] inputTypes)
+    {
+        var newTokenListList = new List<List<IToken>>();
+        
+        foreach(var tokenList in this.rawTokens)
+        {
+            var newTokenList = new List<IToken>();
+            foreach(var token in tokenList)
+            {
+                if(inputTypes.Contains(token.GetInputType()))
+                {
+                    newTokenList.Add(token);
+                }
+            }
+            newTokenListList.Add(newTokenList);
+        }
+
+        this.rawTokens = newTokenListList;
+
+        return this;
+    }
 }

TextParser/Schema/BlockType.cs

@@ -4,10 +4,13 @@
 public enum BlockType
 {
     Integer = 1,
-    String = 2,
+    Char = 2,
+    String = 4,
     // technically not a block type but keeping it here for consistency/having all types in one place
-    Fragment = 4,
-    FixedRepetition = 8,
-    GreedyRepetition = 16,
-    NonZeroRepetition = 32,
+    Fragment = 8,
+    FixedRepetition = 16,
+    GreedyRepetition = 32,
+    NonZeroRepetition = 64,
+    Custom = 128,
+    Long = 256,
 }

TextParser/Schema/BuildingBlocks/BuildingBlockBase.cs

@@ -8,7 +8,7 @@ abstract class BuildingBlockBase : IBuildingBlock
     {
     }
 
-    public abstract IToken ParseWord(InputProvider inputs);
+    public abstract List<IToken> ParseWord(InputProvider inputs);
 
     public abstract bool CanParseWord(InputProvider inputs);
 

TextParser/Schema/BuildingBlocks/CharBlock.cs

@@ -0,0 +1,49 @@
+namespace Parsing.Schema.BuildingBlocks;
+
+using Parsing.Tokenization;
+
+class CharBlock : BuildingBlockBase
+{
+
+    public CharBlock()
+    {
+    }
+
+    public override List<IToken> ParseWord(InputProvider inputs)
+    {
+        var tokenList = new List<IToken>();
+        foreach (char c in inputs.YieldWord())
+        {
+            tokenList.Add(new StringToken(c.ToString()));
+        }
+        return tokenList;
+    }
+
+    public override bool CanParseWord(InputProvider inputs)
+    {
+        string word = string.Empty;
+        using (inputs.GetLookaheadContext())
+        {
+            word = inputs.YieldWord();
+        }
+
+        return this.CanParseWord(word);
+    }
+
+    public override bool CanParseWord(string word)
+    {
+        // Here we need to ensure we are not matching any non-string tokens, since string can match pretty much anything
+        IntegerBlock intBlock = new IntegerBlock();
+        if (intBlock.CanParseWord(word))
+        {
+            return false;
+        }
+
+        return true;
+    }
+
+    public override BlockType GetBlockType()
+    {
+        return BlockType.String;
+    }
+}

TextParser/Schema/BuildingBlocks/CustomInputBlock.cs

@@ -0,0 +1,42 @@
+namespace Parsing.Schema.BuildingBlocks;
+
+using Parsing.Tokenization;
+
+class CustomInputBlock<T> : BuildingBlockBase
+{
+
+    private InputType definedInputType;
+    private Func<string, T> wordConverter;
+
+    public CustomInputBlock(InputType definedInputType, Func<string, T> wordConverter)
+    {
+        this.definedInputType = definedInputType;
+        this.wordConverter = wordConverter;
+    }
+
+    public override List<IToken> ParseWord(InputProvider inputs)
+    {
+        return new List<IToken>() { new CustomToken<T>(inputs.YieldWord(), this.definedInputType, this.wordConverter) };
+    }
+
+    public override bool CanParseWord(InputProvider inputs)
+    {
+        string word = string.Empty;
+        using (inputs.GetLookaheadContext())
+        {
+            word = inputs.YieldWord();
+        }
+
+        return this.CanParseWord(word);
+    }
+
+    public override bool CanParseWord(string word)
+    {
+        return true;
+    }
+
+    public override BlockType GetBlockType()
+    {
+        return BlockType.Custom;
+    }
+}

TextParser/Schema/BuildingBlocks/FixedRepetitionBlock.cs

@@ -20,7 +20,7 @@ class FixedRepetitionBlock : BuildingBlockBase
         this.context = this.inputSchema.CreateContext();
     }
 
-    public override IToken ParseWord(InputProvider inputs)
+    public override List<IToken> ParseWord(InputProvider inputs)
     {
         var result = inputSchema.ProcessNextWord(context, inputs);
         if (context.HasFinished)
@@ -31,7 +31,7 @@ class FixedRepetitionBlock : BuildingBlockBase
                 this.context = this.inputSchema.CreateContext();
             }
         }
-        return result.Single();
+        return result;
     }
 
     public override bool CanParseWord(InputProvider inputs)

TextParser/Schema/BuildingBlocks/GreedyRepetitionBlock.cs

@@ -15,14 +15,14 @@ class GreedyRepetitionBlock : BuildingBlockBase
         this.context = this.inputSchema.CreateContext();
     }
 
-    public override IToken ParseWord(InputProvider inputs)
+    public override List<IToken> ParseWord(InputProvider inputs)
     {
         var result = inputSchema.ProcessNextWord(context, inputs);
         if (!this.CanParseWord(inputs))
         {
             this.context = this.inputSchema.CreateContext();
         }
-        return result.Single();
+        return result;
     }
 
     public override bool CanParseWord(InputProvider inputs)

TextParser/Schema/BuildingBlocks/IBuildingBlock.cs

@@ -4,7 +4,7 @@ using Parsing.Tokenization;
 
 public interface IBuildingBlock
 {
-    public IToken ParseWord(InputProvider inputs);
+    public List<IToken> ParseWord(InputProvider inputs);
 
     public bool CanParseWord(InputProvider inputs);
 

TextParser/Schema/BuildingBlocks/IntegerBlock.cs

@@ -9,9 +9,9 @@ class IntegerBlock : BuildingBlockBase
     {
     }
 
-    public override IToken ParseWord(InputProvider inputs)
+    public override List<IToken> ParseWord(InputProvider inputs)
     {
-        return new IntegerToken(inputs.YieldWord());
+        return new List<IToken>() { new IntegerToken(inputs.YieldWord()) };
     }
 
     public override bool CanParseWord(InputProvider inputs)

TextParser/Schema/BuildingBlocks/LongBlock.cs

@@ -0,0 +1,35 @@
+namespace Parsing.Schema.BuildingBlocks;
+
+using Parsing.Tokenization;
+
+class LongBlock : BuildingBlockBase
+{
+
+    public LongBlock()
+    {
+    }
+
+    public override List<IToken> ParseWord(InputProvider inputs)
+    {
+        return new List<IToken>() { new LongToken(inputs.YieldWord()) };
+    }
+
+    public override bool CanParseWord(InputProvider inputs)
+    {
+        using (inputs.GetLookaheadContext())
+        {
+            return this.CanParseWord(inputs.YieldWord());
+        }
+    }
+
+    public override bool CanParseWord(string word)
+    {
+        long number = 0;
+        return long.TryParse(word, out number);
+    }
+
+    public override BlockType GetBlockType()
+    {
+        return BlockType.Long;
+    }
+}

TextParser/Schema/BuildingBlocks/StringBlock.cs

@@ -9,9 +9,9 @@ class StringBlock : BuildingBlockBase
     {
     }
 
-    public override IToken ParseWord(InputProvider inputs)
+    public override List<IToken> ParseWord(InputProvider inputs)
     {
-        return new StringToken(inputs.YieldWord());
+        return new List<IToken>() { new StringToken(inputs.YieldWord()) };
     }
 
     public override bool CanParseWord(InputProvider inputs)
@@ -28,6 +28,12 @@ class StringBlock : BuildingBlockBase
     public override bool CanParseWord(string word)
     {
         // Here we need to ensure we are not matching any non-string tokens, since string can match pretty much anything
+        LongBlock longBlock = new LongBlock();
+        if (longBlock.CanParseWord(word))
+        {
+            return false;
+        }
+
         IntegerBlock intBlock = new IntegerBlock();
         if (intBlock.CanParseWord(word))
         {

TextParser/Schema/InputSchema.cs

@@ -28,15 +28,13 @@ public class InputSchema : ISchema<InputSchemaContext>
     public List<IToken> ProcessNextWord(InputSchemaContext currentContext, InputProvider inputs)
     {
         var nextBlock = this.buildingBlocks[currentContext.lastProcessedBlockIndex];
-        var token = nextBlock.ParseWord(inputs);
+        var tokens = nextBlock.ParseWord(inputs);
         if (!nextBlock.IsRepetitionType() || nextBlock.CheckIsDoneParsingAndReset(inputs))
         {
             currentContext.lastProcessedBlockIndex++;
             currentContext.HasFinished = currentContext.lastProcessedBlockIndex >= this.buildingBlocks.Count;
         }
-        var newTokenList = new List<IToken>();
-        newTokenList.Add(token);
-        return newTokenList;
+        return tokens;
     }
 
     public bool CanProcessNextWord(InputSchemaContext currentContext, InputProvider inputs)

TextParser/Schema/InputSchemaBuilder.cs

@@ -21,6 +21,27 @@ public class InputSchemaBuilder : RepetitionSchemaBuilder<InputSchemaBuilder, In
             case InputType.Integer:
                 block = new IntegerBlock();
                 break;
+            case InputType.Long:
+                block = new LongBlock();
+                break;
+            case InputType.Char:
+                block = new CharBlock();
+                break;
+            default:
+                throw new Exception("Unrecognized InputType");
+        }
+        schema.AddBuildingBlock(block);
+        return this;
+    }
+
+    public InputSchemaBuilder Expect<T>(InputType type, InputType definedInputType, Func<string, T> wordConverter)
+    {
+        IBuildingBlock block;
+        switch (type)
+        {
+            case InputType.Custom:
+                block = new CustomInputBlock<T>(definedInputType, wordConverter);
+                break;
             default:
                 throw new Exception("Unrecognized InputType");
         }

TextParser/Schema/InputType.cs

@@ -6,4 +6,7 @@ public enum InputType
     Integer = BlockType.Integer,
     String = BlockType.String,
     Fragment = BlockType.Fragment,
+    Char = BlockType.Char,
+    Custom = BlockType.Custom,
+    Long = BlockType.Long,
 }

TextParser/Tokenization/CustomToken.cs

@@ -0,0 +1,34 @@
+namespace Parsing.Tokenization;
+
+using Parsing.Schema;
+
+public class CustomToken<T> : IValueToken<T>
+{
+    private string word;
+
+    private InputType definedInputType;
+
+    private Func<string, T> wordConverter;
+
+    public CustomToken(string word, InputType definedInputType, Func<string, T> wordConverter)
+    {
+        this.word = word;
+        this.wordConverter = wordConverter;
+        this.definedInputType = definedInputType;
+    }
+
+    public string GetText()
+    {
+        return word;
+    }
+
+    public T GetValue()
+    {
+        return wordConverter(word);
+    }
+
+    public InputType GetInputType()
+    {
+        return this.definedInputType;
+    }
+}

TextParser/Tokenization/LongToken.cs

@@ -0,0 +1,28 @@
+namespace Parsing.Tokenization;
+
+using Parsing.Schema;
+
+public class LongToken : IValueToken<long>
+{
+    private string word;
+
+    public LongToken(string word)
+    {
+        this.word = word;
+    }
+
+    public string GetText()
+    {
+        return word;
+    }
+
+    public long GetValue()
+    {
+        return long.Parse(word);
+    }
+
+    public InputType GetInputType()
+    {
+        return InputType.Long;
+    }
+}

TextParser/VERSION

@@ -1 +1 @@
-0.7.0
+0.10.0