Compare commits
29 Commits
0.9.1
...
a630cac9a1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a630cac9a1 | ||
|
|
8475389b99 | ||
| 1d68b70fa5 | |||
| e41d93e301 | |||
| 19fdbe4dac | |||
| ec1e65c03b | |||
| f6701df12d | |||
| 610b628081 | |||
| f480334eee | |||
| 8608e4ddd1 | |||
| 7133ec1560 | |||
| 2f92f59a50 | |||
| 97a4da7531 | |||
| c577f15f91 | |||
| b4d55ddfc0 | |||
| f51aad91fa | |||
| cab7ac35ae | |||
| 86bb120e32 | |||
| c728509978 | |||
| 7bd2429a43 | |||
| d41d9f1791 | |||
| 1a8a82c106 | |||
| 414c74be84 | |||
| f8c00da2b8 | |||
| d4ac62c592 | |||
| 0f533c2018 | |||
| 2067fe06fc | |||
| cc0f0a24d9 | |||
| c41d665ab8 |
@@ -57,12 +57,6 @@ jobs:
|
||||
run: |
|
||||
REPOSITORY_OWNER=$(echo "$GITHUB_REPOSITORY" | awk -F '/' '{print $1}')
|
||||
REPOSITORY_NAME=$(echo "$GITHUB_REPOSITORY" | awk -F '/' '{print $2}' | tr '-' '_')
|
||||
REPOSITORY_SOURCE_NAME=gitea-repo
|
||||
if [ -z "$(dotnet nuget config get all | grep "/packages/${REPOSITORY_OWNER}/nuget/index.json")" ]; then
|
||||
dotnet nuget add source --name $REPOSITORY_SOURCE_NAME https://git.disi.dev/api/packages/$REPOSITORY_OWNER/nuget/index.json
|
||||
else
|
||||
QUOTED_REPOSITORY_SOURCE_NAME=$(dotnet nuget config get all | grep "/packages/${REPOSITORY_OWNER}/nuget/index.json" | awk '{print $2}' | awk -F= '{print $2}')
|
||||
REPOSITORY_SOURCE_NAME=${QUOTED_REPOSITORY_SOURCE_NAME:1:-1}
|
||||
fi
|
||||
dotnet pack --include-symbols --include-source -p:PackageVersion=$(cat $REPOSITORY_NAME/VERSION) TextParser.sln
|
||||
dotnet nuget push -k $GITEA_PAT --source $REPOSITORY_SOURCE_NAME $REPOSITORY_NAME/bin/Release/$REPOSITORY_NAME.$(cat $REPOSITORY_NAME/VERSION).symbols.nupkg
|
||||
echo "dotnet nuget push -k ${GITEA_PAT:0:-15} $REPOSITORY_NAME/bin/Release/$REPOSITORY_NAME.$(cat $REPOSITORY_NAME/VERSION).symbols.nupkg"
|
||||
dotnet nuget push -k $GITEA_PAT $REPOSITORY_NAME/bin/Release/$REPOSITORY_NAME.$(cat $REPOSITORY_NAME/VERSION).symbols.nupkg
|
||||
|
||||
149
HISTORY.md
149
HISTORY.md
@@ -1,149 +0,0 @@
|
||||
Changelog
|
||||
=========
|
||||
|
||||
|
||||
(unreleased)
|
||||
------------
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Fix build issues, ref: NOISSUE. [Simon Diesenreiter]
|
||||
|
||||
|
||||
0.9.0 (2024-12-13)
|
||||
------------------
|
||||
- Feat: add filter option to TokenConverter, ref: NOISSUE. [Simon
|
||||
Diesenreiter]
|
||||
|
||||
|
||||
0.8.0 (2024-12-12)
|
||||
------------------
|
||||
- Feat: adding sensible index constructors refs: NOISSUE. [Simon
|
||||
Diesenreiter]
|
||||
|
||||
|
||||
0.7.2 (2024-12-05)
|
||||
------------------
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Add some missing API methods, ref: NOISSUE. [Simon Diesenreiter]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
|
||||
|
||||
0.7.1 (2024-12-05)
|
||||
------------------
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Allow for parsing single chars as input, ref: NOISSUE. [Simon
|
||||
Diesenreiter]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
|
||||
|
||||
0.7.0 (2024-12-05)
|
||||
------------------
|
||||
- Feat: implement generic data set manipulator, ref: NOISSUE. [Simon
|
||||
Diesenreiter]
|
||||
|
||||
|
||||
0.6.0 (2024-12-03)
|
||||
------------------
|
||||
- Feat: enable named literals, ref: NOISSUE. [Simon Diesenreiter]
|
||||
|
||||
|
||||
0.5.1 (2024-12-03)
|
||||
------------------
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Some bugfixes with fragment parser logic, ref: NOISSUE. [Simon
|
||||
Diesenreiter]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
|
||||
|
||||
0.5.0 (2024-12-03)
|
||||
------------------
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Fix bugs with fragment parsing support, refs: NOISSUE. [Simon
|
||||
Diesenreiter]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
- Feat: initial support for fragment parsing, ref: NOISSUE. [Simon
|
||||
Diesenreiter]
|
||||
|
||||
|
||||
0.4.0 (2024-12-02)
|
||||
------------------
|
||||
- Style: fix linting errors, ref: NOISSUE. [Simon Diesenreiter]
|
||||
- Feat: implement greedy repetition, ref: A24-13. [Simon Diesenreiter]
|
||||
|
||||
|
||||
0.3.0 (2024-12-02)
|
||||
------------------
|
||||
- Feat: ci pipeline fix for releases, ref NOISSUE. [Simon Diesenreiter]
|
||||
|
||||
|
||||
0.2.3 (2024-12-02)
|
||||
------------------
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- More release script fixes,ref: NOISSUE. [Simon Diesenreiter]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
|
||||
|
||||
0.2.2 (2024-12-02)
|
||||
------------------
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Increment version,refs:NOISSUE. [Simon Diesenreiter]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
|
||||
|
||||
0.2.1 (2024-12-02)
|
||||
------------------
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Fix file inclusion in tag commits, ref: NOISSUE. [Simon Diesenreiter]
|
||||
|
||||
|
||||
0.2.0 (2024-12-02)
|
||||
------------------
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Fix some bugs and tests, ref: A24-3. [Simon Diesenreiter]
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
- Ci: fix release shortcut, ref: NOISSUE. [Simon Diesenreiter]
|
||||
- Feat: add text parser and output format options, ref: A24-3. [Simon
|
||||
Diesenreiter]
|
||||
|
||||
|
||||
0.1.0 (2024-12-01)
|
||||
------------------
|
||||
- Feat: added initial implementation of TextParser, ref: A24-3. [Simon
|
||||
Diesenreiter]
|
||||
- ✅ Ready to clone and code. [simon]
|
||||
- Ci: initial commit for triggering migration, ref: NOISSUE. [Simon
|
||||
Diesenreiter]
|
||||
- Initial commit. [Projects <>]
|
||||
|
||||
|
||||
|
||||
@@ -33,6 +33,8 @@ public class TextParserTests
|
||||
private const string testInput10 = @"abc
|
||||
bca
|
||||
cab";
|
||||
private const string testInput11 = @"2 x y 4 x y 6 x y 4 x y 1 x y";
|
||||
private const string testInput12 = @"abcd";
|
||||
|
||||
[Fact]
|
||||
public void LineParser_TestSimpleRepetition()
|
||||
@@ -394,7 +396,7 @@ public class TextParserTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TextPArser_TestReadingChars()
|
||||
public void TextParser_TestReadingChars()
|
||||
{
|
||||
var schemaBuilder = new InputSchemaBuilder();
|
||||
var schema = schemaBuilder
|
||||
@@ -415,4 +417,54 @@ public class TextParserTests
|
||||
Assert.Equal(3, row[1].Count);
|
||||
Assert.Equal(3, row[2].Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TextParser_TestFilter()
|
||||
{
|
||||
var schemaBuilder = new InputSchemaBuilder();
|
||||
var schema = schemaBuilder
|
||||
.Repeat()
|
||||
.Expect(InputType.Integer)
|
||||
.Expect(InputType.Char)
|
||||
.Expect(InputType.Char)
|
||||
.EndRepetition()
|
||||
.Build();
|
||||
|
||||
var parser = new TextParser<InputSchemaContext>(schema);
|
||||
var numbers = parser
|
||||
.SetInputText(testInput11)
|
||||
.Parse()
|
||||
.Filter(InputType.Integer)
|
||||
.AsSingleStream<int>();
|
||||
|
||||
Assert.Equal(5, numbers.Count);
|
||||
Assert.Equal(2, numbers[0]);
|
||||
Assert.Equal(4, numbers[1]);
|
||||
Assert.Equal(6, numbers[2]);
|
||||
Assert.Equal(4, numbers[3]);
|
||||
Assert.Equal(1, numbers[4]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TextParser_TestCharExplicit()
|
||||
{
|
||||
var schemaBuilder = new InputSchemaBuilder();
|
||||
var schema = schemaBuilder
|
||||
.Repeat()
|
||||
.Expect(InputType.Char)
|
||||
.EndRepetition()
|
||||
.Build();
|
||||
|
||||
var parser = new TextParser<InputSchemaContext>(schema);
|
||||
var numbers = parser
|
||||
.SetInputText(testInput12)
|
||||
.Parse()
|
||||
.AsSingleStream<string>();
|
||||
|
||||
Assert.Equal(4, numbers.Count);
|
||||
Assert.Equal("a", numbers[0]);
|
||||
Assert.Equal("b", numbers[1]);
|
||||
Assert.Equal("c", numbers[2]);
|
||||
Assert.Equal("d", numbers[3]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ public class TokenConverter
|
||||
{
|
||||
}
|
||||
|
||||
private List<T> AsGenericCollection<T, U>() where T : ICollection<U>, new()
|
||||
private List<T> AsGenericCollection<T, U>() where T : List<U>, new()
|
||||
{
|
||||
List<T> returnData = new List<T>();
|
||||
foreach (var tokenRow in this.rawTokens)
|
||||
@@ -25,11 +25,15 @@ public class TokenConverter
|
||||
{
|
||||
throw new Exception("No token was provided, but token was expected!");
|
||||
}
|
||||
IValueToken<U>? valueToken = token as IValueToken<U>;
|
||||
if (valueToken == null)
|
||||
|
||||
if (!token.GetType().IsAssignableTo(typeof(IValueToken<U>)))
|
||||
{
|
||||
throw new Exception("Provided token is not a ValueToken");
|
||||
Console.WriteLine(token.GetText());
|
||||
Type t = token.GetType();
|
||||
throw new Exception("Provided token is not a ValueToken - type: " + t.ToString());
|
||||
}
|
||||
|
||||
IValueToken<U> valueToken = token as IValueToken<U>;
|
||||
newRow.Add(valueToken.GetValue());
|
||||
}
|
||||
|
||||
@@ -157,11 +161,11 @@ public class TokenConverter
|
||||
return newList;
|
||||
}
|
||||
|
||||
public TokenConverter Filter<T>(params InputType[] inputTypes)
|
||||
public TokenConverter Filter(params InputType[] inputTypes)
|
||||
{
|
||||
var newTokenListList = new List<List<IToken>>();
|
||||
|
||||
foreach(var tokenList in rawTokens)
|
||||
foreach(var tokenList in this.rawTokens)
|
||||
{
|
||||
var newTokenList = new List<IToken>();
|
||||
foreach(var token in tokenList)
|
||||
|
||||
@@ -11,4 +11,6 @@ public enum BlockType
|
||||
FixedRepetition = 16,
|
||||
GreedyRepetition = 32,
|
||||
NonZeroRepetition = 64,
|
||||
Custom = 128,
|
||||
Long = 256,
|
||||
}
|
||||
|
||||
@@ -44,6 +44,6 @@ class CharBlock : BuildingBlockBase
|
||||
|
||||
public override BlockType GetBlockType()
|
||||
{
|
||||
return BlockType.String;
|
||||
return BlockType.Char;
|
||||
}
|
||||
}
|
||||
42
TextParser/Schema/BuildingBlocks/CustomInputBlock.cs
Normal file
42
TextParser/Schema/BuildingBlocks/CustomInputBlock.cs
Normal file
@@ -0,0 +1,42 @@
|
||||
namespace Parsing.Schema.BuildingBlocks;
|
||||
|
||||
using Parsing.Tokenization;
|
||||
|
||||
class CustomInputBlock<T> : BuildingBlockBase
|
||||
{
|
||||
|
||||
private InputType definedInputType;
|
||||
private Func<string, T> wordConverter;
|
||||
|
||||
public CustomInputBlock(InputType definedInputType, Func<string, T> wordConverter)
|
||||
{
|
||||
this.definedInputType = definedInputType;
|
||||
this.wordConverter = wordConverter;
|
||||
}
|
||||
|
||||
public override List<IToken> ParseWord(InputProvider inputs)
|
||||
{
|
||||
return new List<IToken>() { new CustomToken<T>(inputs.YieldWord(), this.definedInputType, this.wordConverter) };
|
||||
}
|
||||
|
||||
public override bool CanParseWord(InputProvider inputs)
|
||||
{
|
||||
string word = string.Empty;
|
||||
using (inputs.GetLookaheadContext())
|
||||
{
|
||||
word = inputs.YieldWord();
|
||||
}
|
||||
|
||||
return this.CanParseWord(word);
|
||||
}
|
||||
|
||||
public override bool CanParseWord(string word)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
public override BlockType GetBlockType()
|
||||
{
|
||||
return BlockType.Custom;
|
||||
}
|
||||
}
|
||||
35
TextParser/Schema/BuildingBlocks/LongBlock.cs
Normal file
35
TextParser/Schema/BuildingBlocks/LongBlock.cs
Normal file
@@ -0,0 +1,35 @@
|
||||
namespace Parsing.Schema.BuildingBlocks;
|
||||
|
||||
using Parsing.Tokenization;
|
||||
|
||||
class LongBlock : BuildingBlockBase
|
||||
{
|
||||
|
||||
public LongBlock()
|
||||
{
|
||||
}
|
||||
|
||||
public override List<IToken> ParseWord(InputProvider inputs)
|
||||
{
|
||||
return new List<IToken>() { new LongToken(inputs.YieldWord()) };
|
||||
}
|
||||
|
||||
public override bool CanParseWord(InputProvider inputs)
|
||||
{
|
||||
using (inputs.GetLookaheadContext())
|
||||
{
|
||||
return this.CanParseWord(inputs.YieldWord());
|
||||
}
|
||||
}
|
||||
|
||||
public override bool CanParseWord(string word)
|
||||
{
|
||||
long number = 0;
|
||||
return long.TryParse(word, out number);
|
||||
}
|
||||
|
||||
public override BlockType GetBlockType()
|
||||
{
|
||||
return BlockType.Long;
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,12 @@ class StringBlock : BuildingBlockBase
|
||||
public override bool CanParseWord(string word)
|
||||
{
|
||||
// Here we need to ensure we are not matching any non-string tokens, since string can match pretty much anything
|
||||
LongBlock longBlock = new LongBlock();
|
||||
if (longBlock.CanParseWord(word))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
IntegerBlock intBlock = new IntegerBlock();
|
||||
if (intBlock.CanParseWord(word))
|
||||
{
|
||||
|
||||
@@ -17,6 +17,7 @@ public class FragmentSchema : ISchema<FragmentSchemaContext>
|
||||
private string fragmentRegex;
|
||||
private List<string> namedGroups = new List<string>();
|
||||
private List<string> namedLiterals = new List<string>();
|
||||
private bool ignoreEmptyMatches = true;
|
||||
|
||||
public FragmentSchema(string fragmentRegex, List<string> namedGroups, List<string> namedLiterals)
|
||||
{
|
||||
@@ -33,6 +34,11 @@ public class FragmentSchema : ISchema<FragmentSchemaContext>
|
||||
// one token per match
|
||||
foreach (Match match in r.Matches(inputs.YieldWord()))
|
||||
{
|
||||
if(this.ignoreEmptyMatches && string.IsNullOrEmpty(match.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var newToken = new FragmentToken(match.Value);
|
||||
// token contains data from all included matches
|
||||
foreach (var groupName in this.namedGroups)
|
||||
@@ -109,4 +115,9 @@ public class FragmentSchema : ISchema<FragmentSchemaContext>
|
||||
{
|
||||
return new FragmentSchemaContext();
|
||||
}
|
||||
|
||||
public void ShouldIgnoreEmptyMatches(bool ignoreEmptyMatches)
|
||||
{
|
||||
this.ignoreEmptyMatches = ignoreEmptyMatches;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,12 +52,22 @@ public class FragmentSchemaBuilder : RepetitionSchemaBuilder<FragmentSchemaBuild
|
||||
case InputType.Integer:
|
||||
this.fragmentRegex += "(" + groupNamePrefix + "\\d+)";
|
||||
break;
|
||||
case InputType.Char:
|
||||
this.fragmentRegex += "(" + groupNamePrefix + "[a-zA-Z])";
|
||||
break;
|
||||
default:
|
||||
throw new Exception("Unrecognized InputType");
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public FragmentSchemaBuilder ShouldIgnoreEmptyMatches(bool ignoreEmptyMatches)
|
||||
{
|
||||
this.ShouldIgnoreEmptyMatches(ignoreEmptyMatches);
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
public FragmentSchemaBuilder Expect(string literal, string name = "")
|
||||
{
|
||||
string groupNamePrefix = "";
|
||||
@@ -130,6 +140,10 @@ public class FragmentSchemaBuilder : RepetitionSchemaBuilder<FragmentSchemaBuild
|
||||
}
|
||||
|
||||
oldSchemaBuilder.fragmentRegex += "(" + currentRegex + ")";
|
||||
var groupsToAdd = currentBuilder.namedGroups.Where(g => !oldSchemaBuilder.namedGroups.Contains(g)).ToList();
|
||||
var literalsToAdd = currentBuilder.namedLiterals.Where(l => !oldSchemaBuilder.namedLiterals.Contains(l)).ToList();
|
||||
oldSchemaBuilder.namedGroups.AddRange(groupsToAdd);
|
||||
oldSchemaBuilder.namedLiterals.AddRange(literalsToAdd);
|
||||
|
||||
return oldSchemaBuilder;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,9 @@ public class InputSchemaBuilder : RepetitionSchemaBuilder<InputSchemaBuilder, In
|
||||
case InputType.Integer:
|
||||
block = new IntegerBlock();
|
||||
break;
|
||||
case InputType.Long:
|
||||
block = new LongBlock();
|
||||
break;
|
||||
case InputType.Char:
|
||||
block = new CharBlock();
|
||||
break;
|
||||
@@ -31,6 +34,21 @@ public class InputSchemaBuilder : RepetitionSchemaBuilder<InputSchemaBuilder, In
|
||||
return this;
|
||||
}
|
||||
|
||||
public InputSchemaBuilder Expect<T>(InputType type, InputType definedInputType, Func<string, T> wordConverter)
|
||||
{
|
||||
IBuildingBlock block;
|
||||
switch (type)
|
||||
{
|
||||
case InputType.Custom:
|
||||
block = new CustomInputBlock<T>(definedInputType, wordConverter);
|
||||
break;
|
||||
default:
|
||||
throw new Exception("Unrecognized InputType");
|
||||
}
|
||||
schema.AddBuildingBlock(block);
|
||||
return this;
|
||||
}
|
||||
|
||||
public InputSchemaBuilder Repeat(int repetitionCount)
|
||||
{
|
||||
// add another layer of parsing
|
||||
|
||||
@@ -7,4 +7,6 @@ public enum InputType
|
||||
String = BlockType.String,
|
||||
Fragment = BlockType.Fragment,
|
||||
Char = BlockType.Char,
|
||||
Custom = BlockType.Custom,
|
||||
Long = BlockType.Long,
|
||||
}
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
namespace Parsing;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using Parsing.Schema;
|
||||
using Parsing.Tokenization;
|
||||
|
||||
public static class DataConversionHelpers
|
||||
{
|
||||
public static List<TNewType> ConvertData<TTokenType, TNewType, TOldType>(this List<IToken> tokenList, Func<TOldType, TNewType> converter) where TTokenType : IValueToken<TOldType>
|
||||
{
|
||||
var newList = new List<TNewType>();
|
||||
foreach (var token in tokenList)
|
||||
{
|
||||
var typedToken = token as IValueToken<TOldType>;
|
||||
if (typedToken == null)
|
||||
{
|
||||
throw new Exception("Invalid Token type encountered during value conversion");
|
||||
}
|
||||
|
||||
newList.Add(converter(typedToken.GetValue()));
|
||||
}
|
||||
return newList;
|
||||
}
|
||||
|
||||
public static List<TNewType> ConvertData<TTokenType, TNewType, TOldType>(this List<IToken> tokenList, Func<TOldType, List<TNewType>> converter) where TTokenType : IValueToken<TOldType>
|
||||
{
|
||||
var newList = new List<TNewType>();
|
||||
foreach (var token in tokenList)
|
||||
{
|
||||
var typedToken = token as IValueToken<TOldType>;
|
||||
if (typedToken == null)
|
||||
{
|
||||
throw new Exception("Invalid Token type encountered during value conversion");
|
||||
}
|
||||
|
||||
newList.AddRange(converter(typedToken.GetValue()));
|
||||
}
|
||||
return newList;
|
||||
}
|
||||
|
||||
public static List<List<TNewType>> ConvertData<TTokenType, TNewType, TOldType>(this List<List<IToken>> tokenListList, Func<TOldType, TNewType> converter) where TTokenType : IValueToken<TOldType>
|
||||
{
|
||||
var newListList = new List<List<TNewType>>();
|
||||
foreach (var tokenList in tokenListList)
|
||||
{
|
||||
newListList.Add(tokenList.ConvertData<TTokenType, TNewType, TOldType>(converter));
|
||||
}
|
||||
return newListList;
|
||||
}
|
||||
}
|
||||
|
||||
public static class DataManipulationHelpers
|
||||
{
|
||||
public static TType ReduceData<TType>(this List<TType> data, Func<TType, TType, TType> reducer)
|
||||
{
|
||||
if (data.Count < 2)
|
||||
{
|
||||
return data[0];
|
||||
}
|
||||
TType result = data[0];
|
||||
for (int i = 1; i < data.Count; i++)
|
||||
{
|
||||
result = reducer(result, data[i]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static TType ReduceData<TType>(this List<TType> data, Func<List<TType>, TType> reducer)
|
||||
{
|
||||
return reducer(data);
|
||||
}
|
||||
}
|
||||
|
||||
public class TokenConverter
|
||||
{
|
||||
protected List<List<IToken>> rawTokens = new List<List<IToken>>();
|
||||
|
||||
public TokenConverter()
|
||||
{
|
||||
}
|
||||
|
||||
private List<T> AsGenericCollection<T, U>() where T : ICollection<U>, new()
|
||||
{
|
||||
List<T> returnData = new List<T>();
|
||||
foreach (var tokenRow in this.rawTokens)
|
||||
{
|
||||
T newRow = new T();
|
||||
foreach (IToken token in tokenRow)
|
||||
{
|
||||
if (token == null)
|
||||
{
|
||||
throw new Exception("No token was provided, but token was expected!");
|
||||
}
|
||||
IValueToken<U>? valueToken = token as IValueToken<U>;
|
||||
if (valueToken == null)
|
||||
{
|
||||
throw new Exception("Provided token is not a ValueToken");
|
||||
}
|
||||
newRow.Add(valueToken.GetValue());
|
||||
}
|
||||
|
||||
returnData.Add(newRow);
|
||||
}
|
||||
return returnData;
|
||||
}
|
||||
|
||||
private void CheckConversionPrerequisites()
|
||||
{
|
||||
// in order to convert rows to columns or grid we require every row to have the same length
|
||||
int rowLength = this.rawTokens[0].Count;
|
||||
|
||||
foreach (var tokenRow in this.rawTokens)
|
||||
{
|
||||
if (tokenRow.Count != rowLength)
|
||||
{
|
||||
throw new Exception("Attempted to convert token dataset that is not able to be converted!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public List<T> AsSingleStream<T>()
|
||||
{
|
||||
List<T> returnData = new List<T>();
|
||||
foreach (var tokenRow in this.rawTokens)
|
||||
{
|
||||
foreach (IToken token in tokenRow)
|
||||
{
|
||||
if (token == null)
|
||||
{
|
||||
throw new Exception("No token was provided, but token was expected!");
|
||||
}
|
||||
IValueToken<T>? valueToken = token as IValueToken<T>;
|
||||
if (valueToken == null)
|
||||
{
|
||||
throw new Exception("Provided token is not a ValueToken");
|
||||
}
|
||||
returnData.Add(valueToken.GetValue());
|
||||
}
|
||||
}
|
||||
return returnData;
|
||||
}
|
||||
|
||||
public List<T[]> AsRows<T>()
|
||||
{
|
||||
var listRows = this.AsListRows<T>();
|
||||
var newList = new List<T[]>();
|
||||
|
||||
foreach (var rowList in listRows)
|
||||
{
|
||||
newList.Add(rowList.ToArray());
|
||||
}
|
||||
|
||||
return newList;
|
||||
}
|
||||
|
||||
public List<List<T>> AsListRows<T>()
|
||||
{
|
||||
return this.AsGenericCollection<List<T>, T>();
|
||||
}
|
||||
|
||||
public List<T[]> AsColumns<T>()
|
||||
{
|
||||
var listColumns = this.AsListColumns<T>();
|
||||
var newList = new List<T[]>();
|
||||
|
||||
foreach (var columnList in listColumns)
|
||||
{
|
||||
newList.Add(columnList.ToArray());
|
||||
}
|
||||
|
||||
return newList;
|
||||
}
|
||||
|
||||
public List<List<T>> AsListColumns<T>()
|
||||
{
|
||||
this.CheckConversionPrerequisites();
|
||||
var rows = AsListRows<T>();
|
||||
|
||||
var columns = new List<List<T>>();
|
||||
for (int i = 0; i < rows[0].Count; i++)
|
||||
{
|
||||
columns.Add(new List<T>());
|
||||
}
|
||||
|
||||
foreach (var row in rows)
|
||||
{
|
||||
for (int i = 0; i < row.Count; i++)
|
||||
{
|
||||
columns[i].Add(row[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return columns;
|
||||
}
|
||||
|
||||
public T[][] AsGrid<T>()
|
||||
{
|
||||
this.CheckConversionPrerequisites();
|
||||
var rowsList = AsRows<T>();
|
||||
return rowsList.ToArray();
|
||||
}
|
||||
|
||||
public List<List<IToken>> AsRawData()
|
||||
{
|
||||
return this.rawTokens;
|
||||
}
|
||||
|
||||
public List<Fragment> AsFragments()
|
||||
{
|
||||
var items = this.AsSingleStream<Fragment>();
|
||||
var newList = new List<Fragment>();
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
var typedItem = item as Fragment;
|
||||
if (typedItem == null)
|
||||
{
|
||||
throw new Exception("Invalid token type encountered");
|
||||
}
|
||||
newList.Add(typedItem);
|
||||
}
|
||||
|
||||
return newList;
|
||||
}
|
||||
}
|
||||
34
TextParser/Tokenization/CustomToken.cs
Normal file
34
TextParser/Tokenization/CustomToken.cs
Normal file
@@ -0,0 +1,34 @@
|
||||
namespace Parsing.Tokenization;
|
||||
|
||||
using Parsing.Schema;
|
||||
|
||||
public class CustomToken<T> : IValueToken<T>
|
||||
{
|
||||
private string word;
|
||||
|
||||
private InputType definedInputType;
|
||||
|
||||
private Func<string, T> wordConverter;
|
||||
|
||||
public CustomToken(string word, InputType definedInputType, Func<string, T> wordConverter)
|
||||
{
|
||||
this.word = word;
|
||||
this.wordConverter = wordConverter;
|
||||
this.definedInputType = definedInputType;
|
||||
}
|
||||
|
||||
public string GetText()
|
||||
{
|
||||
return word;
|
||||
}
|
||||
|
||||
public T GetValue()
|
||||
{
|
||||
return wordConverter(word);
|
||||
}
|
||||
|
||||
public InputType GetInputType()
|
||||
{
|
||||
return this.definedInputType;
|
||||
}
|
||||
}
|
||||
28
TextParser/Tokenization/LongToken.cs
Normal file
28
TextParser/Tokenization/LongToken.cs
Normal file
@@ -0,0 +1,28 @@
|
||||
namespace Parsing.Tokenization;
|
||||
|
||||
using Parsing.Schema;
|
||||
|
||||
public class LongToken : IValueToken<long>
|
||||
{
|
||||
private string word;
|
||||
|
||||
public LongToken(string word)
|
||||
{
|
||||
this.word = word;
|
||||
}
|
||||
|
||||
public string GetText()
|
||||
{
|
||||
return word;
|
||||
}
|
||||
|
||||
public long GetValue()
|
||||
{
|
||||
return long.Parse(word);
|
||||
}
|
||||
|
||||
public InputType GetInputType()
|
||||
{
|
||||
return InputType.Long;
|
||||
}
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
0.9.1
|
||||
0.12.0
|
||||
|
||||
@@ -8,4 +8,8 @@
|
||||
<add key="gitea-homelab" value="https://git.disi.dev/api/packages/Homelab/nuget/index.json" />
|
||||
<add key="gitea-artifacts" value="https://git.disi.dev/api/packages/Artifacts/nuget/index.json" />
|
||||
</packageSources>
|
||||
|
||||
<config>
|
||||
<add key="DefaultPushSource" value="https://git.disi.dev/api/packages/Projects/nuget/index.json" />
|
||||
</config>
|
||||
</configuration>
|
||||
|
||||
Reference in New Issue
Block a user