feat: add filter option to TokenConverter, ref: NOISSUE

TextParser/Data/DataManipulationHelpers.cs

@@ -20,4 +20,24 @@ public static class DataManipulationHelpers
     {
         return reducer(data);
     }
+
+    public static List<TNewType> TransformData<TType, TNewType>(this List<TType> data, Func<TType, TNewType> transformer)
+    {
+        var newList = new List<TNewType>();
+        foreach (TType dataItem in data)
+        {
+            newList.Add(transformer(dataItem));
+        }
+        return newList;
+    }
+
+    public static List<TNewType> TransformData<TType, TNewType>(this List<List<TType>> data, Func<List<TType>, TNewType> transformer)
+    {
+        var newList = new List<TNewType>();
+        foreach (List<TType> dataItemList in data)
+        {
+            newList.Add(transformer(dataItem));
+        }
+        return newList;
+    }
 }

TextParser/Data/TokenConverter.cs

@@ -156,4 +156,21 @@ public class TokenConverter
 
         return newList;
     }
+
+    public TokenConverter Filter<T>(params InputType[] inputTypes)
+    {
+        var newTokenList = new List<List<IToken>>()
+        
+        foreach(var token in rawTokens)
+        {
+            if(inputTypes.Contains(token.GetInputType()))
+            {
+                newTokenList.Add(token);
+            }
+        }
+
+        this.rawTokens = newTokenList;
+
+        return this;
+    }
 }