From b78ab6614dbc12fd88712f2d59e70da6948e29b2 Mon Sep 17 00:00:00 2001
From: Michael Krisper <michael.krisper@tugraz.at>
Date: Tue, 2 Aug 2016 17:07:56 +0200
Subject: [PATCH] CSV File: Robust CSV Reader (from VB.NET), Updated Tests for
Aux Electric System
---
.../Models/Declaration/ElectricSystem.cs | 11 +-
VectoCore/VectoCore/Utils/StreamExtensions.cs | 18 +++
VectoCore/VectoCore/Utils/VectoCSVFile.cs | 103 +++++++-----------
VectoCore/VectoCore/VectoCore.csproj | 2 +
.../VectoCoreTest/FileIO/VectoCSVFileTest.cs | 39 +++++++
.../Models/Declaration/DeclarationDataTest.cs | 67 ++++--------
6 files changed, 128 insertions(+), 112 deletions(-)
create mode 100644 VectoCore/VectoCore/Utils/StreamExtensions.cs
diff --git a/VectoCore/VectoCore/Models/Declaration/ElectricSystem.cs b/VectoCore/VectoCore/Models/Declaration/ElectricSystem.cs
index 5293685895..62943196d5 100644
--- a/VectoCore/VectoCore/Models/Declaration/ElectricSystem.cs
+++ b/VectoCore/VectoCore/Models/Declaration/ElectricSystem.cs
@@ -56,14 +56,17 @@ namespace TUGraz.VectoCore.Models.Declaration
NormalizeTable(table);
foreach (DataRow row in table.Rows) {
- var name = row.Field<string>("Technology");
- foreach (MissionType mission in Enum.GetValues(typeof(MissionType))) {
- Data[Tuple.Create(mission, name)] = row.ParseDouble(mission.ToString().ToLower()).SI<Watt>();
+ var name = row.Field<string>("technology");
+ foreach (DataColumn col in table.Columns) {
+ if (col.Caption != "technology") {
+ Data[Tuple.Create(col.Caption.ParseEnum<MissionType>(), name)] =
+ row.ParseDouble(col).SI<Watt>();
+ }
}
}
}
- public override Watt Lookup(MissionType missionType, string technology)
+ public override Watt Lookup(MissionType missionType, string technology = "Standard technology")
{
var value = base.Lookup(missionType, technology);
return value / _alternator.Lookup(missionType);
diff --git a/VectoCore/VectoCore/Utils/StreamExtensions.cs b/VectoCore/VectoCore/Utils/StreamExtensions.cs
new file mode 100644
index 0000000000..7ff3dff43a
--- /dev/null
+++ b/VectoCore/VectoCore/Utils/StreamExtensions.cs
@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace TUGraz.VectoCore.Utils
+{
+ internal static class StreamExtensions
+ {
+ public static IEnumerable<string> ReadLines(this Stream stream)
+ {
+ using (var reader = new StreamReader(stream, Encoding.UTF8)) {
+ while (!reader.EndOfStream) {
+ yield return reader.ReadLine();
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/VectoCore/VectoCore/Utils/VectoCSVFile.cs b/VectoCore/VectoCore/Utils/VectoCSVFile.cs
index 5c9f77ca31..0f46cd2a54 100644
--- a/VectoCore/VectoCore/Utils/VectoCSVFile.cs
+++ b/VectoCore/VectoCore/Utils/VectoCSVFile.cs
@@ -37,6 +37,7 @@ using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
+using Microsoft.VisualBasic.FileIO;
using TUGraz.VectoCommon.Exceptions;
using TUGraz.VectoCommon.Models;
using TUGraz.VectoCommon.Utils;
@@ -59,8 +60,8 @@ namespace TUGraz.VectoCore.Utils
public static class VectoCSVFile
{
private static readonly Regex HeaderFilter = new Regex(@"\[.*?\]|\<|\>", RegexOptions.Compiled);
- private const char Delimiter = ',';
- private const char Comment = '#';
+ private const string Delimiter = ",";
+ private const string Comment = "#";
/// <summary>
/// Reads a CSV file which is stored in Vecto-CSV-Format.
@@ -90,84 +91,59 @@ namespace TUGraz.VectoCore.Utils
/// <returns>A DataTable which represents the CSV File.</returns>
public static DataTable ReadStream(Stream stream, bool ignoreEmptyColumns = false, bool fullHeader = false)
{
- try {
- return ReadData(ReadLines(stream), ignoreEmptyColumns, fullHeader);
- } catch (Exception e) {
- LogManager.GetLogger(typeof(VectoCSVFile).FullName).Error(e);
- throw new VectoException("Failed to read stream: " + e.Message, e);
- }
- }
+ var p = new TextFieldParser(stream) {
+ TextFieldType = FieldType.Delimited,
+ Delimiters = new[] { Delimiter },
+ CommentTokens = new[] { Comment },
+ HasFieldsEnclosedInQuotes = true,
+ TrimWhiteSpace = true
+ };
- private static IEnumerable<string> ReadLines(Stream stream)
- {
- using (var reader = new StreamReader(stream, Encoding.UTF8)) {
- while (!reader.EndOfStream) {
- yield return reader.ReadLine();
- }
- }
- }
+ string[] colsWithoutComment;
- /// <summary>
- ///
- /// </summary>
- /// <param name="allLines"></param>
- /// <param name="ignoreEmptyColumns"></param>
- /// <param name="fullHeader"></param>
- /// <returns></returns>
- private static DataTable ReadData(IEnumerable<string> allLines, bool ignoreEmptyColumns = false,
- bool fullHeader = false)
- {
- // trim, remove comments and filter empty lines
- var lines = allLines
- .Select(l => l.Trim())
- .Select(l => l.Contains(Comment) ? l.Substring(0, l.IndexOf(Comment)) : l)
- .Where(l => !string.IsNullOrWhiteSpace(l))
- .GetEnumerator();
-
- // start the enumerable
- lines.MoveNext();
-
- // add columns
- var line = lines.Current;
- if (!fullHeader) {
- line = HeaderFilter.Replace(line, "");
+ try {
+ colsWithoutComment = p.ReadFields()
+ .Select(l => l.Contains(Comment) ? l.Substring(0, l.IndexOf(Comment)) : l)
+ .ToArray();
+ } catch (ArgumentNullException) {
+ throw new CSVReadException("CSV Read Error: File was empty.");
}
- double tmp;
- var splittedColumns = line
- .Split(Delimiter);
- var columns = splittedColumns
- .Select(col => col.Trim())
+ double tmp;
+ var columns = colsWithoutComment
+ .Select(l => fullHeader ? l : HeaderFilter.Replace(l, ""))
+ .Select(l => l.Trim())
.Where(col => !double.TryParse(col, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp))
.ToList();
- if (columns.Count > 0) {
- // first line was a valid header: advance to first data line
- lines.MoveNext();
- } else {
+ var firstLineIsData = columns.Count == 0;
+
+ if (firstLineIsData) {
LogManager.GetLogger(typeof(VectoCSVFile).FullName)
.Warn("No valid Data Header found. Interpreting the first line as data line.");
// set the validColumns to: {"0", "1", "2", "3", ...} for all columns in first line.
- columns = splittedColumns.Select((_, index) => index.ToString()).ToList();
+ columns = colsWithoutComment.Select((_, i) => i.ToString()).ToList();
}
var table = new DataTable();
foreach (var col in columns) {
table.Columns.Add(col);
}
- if (lines.Current == null) {
+
+ if (p.EndOfData)
return table;
- }
- // read data into table
- var i = 0;
- do {
- i++;
- line = lines.Current;
- var cells = line.Split(Delimiter).Select(s => s.Trim()).ToArray();
- if (cells.Length != table.Columns.Count && !ignoreEmptyColumns) {
+ do {
+ var cells = firstLineIsData
+ ? colsWithoutComment
+ : p.ReadFields()
+ .Select(l => l.Contains(Comment) ? l.Substring(0, l.IndexOf(Comment)) : l)
+ .Select(s => s.Trim())
+ .ToArray();
+ firstLineIsData = false;
+ if (table.Columns.Count != cells.Length && !ignoreEmptyColumns) {
throw new CSVReadException(
- string.Format("Line {0}: The number of values is not correct. Expected {1} Columns, Got {2} Columns", i,
+ string.Format("Line {0}: The number of values is not correct. Expected {1} Columns, Got {2} Columns", p.LineNumber,
table.Columns.Count, cells.Length));
}
@@ -176,9 +152,10 @@ namespace TUGraz.VectoCore.Utils
table.Rows.Add(cells);
} catch (InvalidCastException e) {
throw new CSVReadException(
- string.Format("Line {0}: The data format of a value is not correct. {1}", i, e.Message), e);
+ string.Format("Line {0}: The data format of a value is not correct. {1}", p.LineNumber, e.Message), e);
}
- } while (lines.MoveNext());
+ } while (!p.EndOfData);
+
return table;
}
diff --git a/VectoCore/VectoCore/VectoCore.csproj b/VectoCore/VectoCore/VectoCore.csproj
index 9fa840bdcc..1161a099e8 100644
--- a/VectoCore/VectoCore/VectoCore.csproj
+++ b/VectoCore/VectoCore/VectoCore.csproj
@@ -87,6 +87,7 @@
<Reference Include="itextsharp">
<HintPath>..\..\packages\iTextSharp.5.5.9\lib\itextsharp.dll</HintPath>
</Reference>
+ <Reference Include="Microsoft.VisualBasic" />
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\packages\Newtonsoft.Json.8.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
@@ -281,6 +282,7 @@
<Compile Include="Models\Simulation\DataBus\IVehicleInfo.cs" />
<Compile Include="Models\Simulation\IVehicleContainer.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
+ <Compile Include="Utils\StreamExtensions.cs" />
<Compile Include="Utils\SwitchExtension.cs" />
<Compile Include="Utils\VectoCSVFile.cs" />
<Compile Include="Utils\DelaunayMap.cs" />
diff --git a/VectoCore/VectoCoreTest/FileIO/VectoCSVFileTest.cs b/VectoCore/VectoCoreTest/FileIO/VectoCSVFileTest.cs
index 450e460d61..aaac221396 100644
--- a/VectoCore/VectoCoreTest/FileIO/VectoCSVFileTest.cs
+++ b/VectoCore/VectoCoreTest/FileIO/VectoCSVFileTest.cs
@@ -78,6 +78,45 @@ namespace TUGraz.VectoCore.Tests.FileIO
CollectionAssert.AreEqual(new[] { "4", "5", "6" }, table.Rows[1].ItemArray);
}
+ [Test]
+ public void VectoCSVFile_ReadStream_Escaped()
+ {
+ var stream = "a,b,c\n\"1,1\",2,3\n4,5,6".GetStream();
+ var table = VectoCSVFile.ReadStream(stream);
+
+ CollectionAssert.AreEqual(new[] { "a", "b", "c" }, table.Columns.Cast<DataColumn>().Select(c => c.ColumnName));
+ Assert.AreEqual(2, table.Rows.Count);
+
+ CollectionAssert.AreEqual(new[] { "1,1", "2", "3" }, table.Rows[0].ItemArray);
+ CollectionAssert.AreEqual(new[] { "4", "5", "6" }, table.Rows[1].ItemArray);
+ }
+
+ [Test]
+ public void VectoCSVFile_ReadStream_Comment()
+ {
+ var stream = "a,b,c\n\"1,1\",2,3#asdf\n4,5,6".GetStream();
+ var table = VectoCSVFile.ReadStream(stream);
+
+ CollectionAssert.AreEqual(new[] { "a", "b", "c" }, table.Columns.Cast<DataColumn>().Select(c => c.ColumnName));
+ Assert.AreEqual(2, table.Rows.Count);
+
+ CollectionAssert.AreEqual(new[] { "1,1", "2", "3" }, table.Rows[0].ItemArray);
+ CollectionAssert.AreEqual(new[] { "4", "5", "6" }, table.Rows[1].ItemArray);
+ }
+
+ [Test]
+ public void VectoCSVFile_ReadStream_EscapedComment()
+ {
+ var stream = "a,b,c\n\"1,1\",2,\"3#asdf\"\n4,5,6".GetStream();
+ var table = VectoCSVFile.ReadStream(stream);
+
+ CollectionAssert.AreEqual(new[] { "a", "b", "c" }, table.Columns.Cast<DataColumn>().Select(c => c.ColumnName));
+ Assert.AreEqual(2, table.Rows.Count);
+
+ CollectionAssert.AreEqual(new[] { "1,1", "2", "3" }, table.Rows[0].ItemArray);
+ CollectionAssert.AreEqual(new[] { "4", "5", "6" }, table.Rows[1].ItemArray);
+ }
+
[Test]
public void VectoCSVFile_ReadStream_No_Header()
{
diff --git a/VectoCore/VectoCoreTest/Models/Declaration/DeclarationDataTest.cs b/VectoCore/VectoCoreTest/Models/Declaration/DeclarationDataTest.cs
index 864997088f..8acfdc7976 100644
--- a/VectoCore/VectoCoreTest/Models/Declaration/DeclarationDataTest.cs
+++ b/VectoCore/VectoCoreTest/Models/Declaration/DeclarationDataTest.cs
@@ -44,7 +44,6 @@ using TUGraz.VectoCore.InputData.Reader.Impl;
using TUGraz.VectoCore.Models.Declaration;
using TUGraz.VectoCore.Models.SimulationComponent.Data;
using TUGraz.VectoCore.Tests.Utils;
-using TUGraz.VectoCore.Utils;
namespace TUGraz.VectoCore.Tests.Models.Declaration
{
@@ -266,52 +265,30 @@ namespace TUGraz.VectoCore.Tests.Models.Declaration
torque.SI<NewtonMeter>() * Math.Pow((angularSpeed / referenceSpeed).Cast<Scalar>(), 2), torqueLookup);
}
- [Test]
- public void AuxElectricSystemTest()
+ [
+ TestCase(MissionType.LongHaul, "Standard technology", 1200, 0.7),
+ TestCase(MissionType.RegionalDelivery, "Standard technology", 1000, 0.7),
+ TestCase(MissionType.UrbanDelivery, "Standard technology", 1000, 0.7),
+ TestCase(MissionType.MunicipalUtility, "Standard technology", 1000, 0.7),
+ TestCase(MissionType.Construction, "Standard technology", 1000, 0.7),
+ TestCase(MissionType.LongHaul, "Standard technology - LED headlights, all", 1150, 0.7),
+ TestCase(MissionType.RegionalDelivery, "Standard technology - LED headlights, all", 950, 0.7),
+ TestCase(MissionType.UrbanDelivery, "Standard technology - LED headlights, all", 950, 0.7),
+ TestCase(MissionType.MunicipalUtility, "Standard technology - LED headlights, all", 950, 0.7),
+ TestCase(MissionType.Construction, "Standard technology - LED headlights, all", 950, 0.7),
+ ]
+ public void AuxElectricSystemTest(MissionType mission, string technology, double value, double efficiency)
{
- var es = DeclarationData.ElectricSystem;
-
- var expected = new[] {
- new { Mission = MissionType.LongHaul, Base = 1240.SI<Watt>(), LED = 1190.SI<Watt>(), Efficiency = 0.7 },
- new {
- Mission = MissionType.RegionalDelivery,
- Base = 1055.SI<Watt>(),
- LED = 1005.SI<Watt>(),
- Efficiency = 0.7
- },
- new {
- Mission = MissionType.UrbanDelivery,
- Base = 974.SI<Watt>(),
- LED = 924.SI<Watt>(),
- Efficiency = 0.7
- },
- new {
- Mission = MissionType.MunicipalUtility,
- Base = 974.SI<Watt>(),
- LED = 924.SI<Watt>(),
- Efficiency = 0.7
- },
- new {
- Mission = MissionType.Construction,
- Base = 975.SI<Watt>(),
- LED = 925.SI<Watt>(),
- Efficiency = 0.7
- },
- new { Mission = MissionType.HeavyUrban, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 },
- new { Mission = MissionType.Urban, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 },
- new { Mission = MissionType.Suburban, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 },
- new { Mission = MissionType.Interurban, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 },
- new { Mission = MissionType.Coach, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 }
- };
- Assert.AreEqual(expected.Length, Enum.GetValues(typeof(MissionType)).Length);
-
- foreach (var expectation in expected) {
- var baseConsumption = es.Lookup(expectation.Mission, null);
- var leds = es.Lookup(expectation.Mission, "LED lights");
+ AssertHelper.AreRelativeEqual(value / efficiency, DeclarationData.ElectricSystem.Lookup(mission, technology));
+ }
- AssertHelper.AreRelativeEqual(expectation.Base / expectation.Efficiency, baseConsumption);
- AssertHelper.AreRelativeEqual(expectation.LED / expectation.Efficiency, leds);
- }
+ [
+ TestCase(MissionType.Interurban, "Standard technology"),
+ TestCase(MissionType.LongHaul, "Standard technology - Flux-Compensator")
+ ]
+ public void AuxElectricSystem_NotExistingError(MissionType mission, string technology)
+ {
+ AssertHelper.Exception<VectoException>(() => { DeclarationData.ElectricSystem.Lookup(mission, technology); });
}
[
--
GitLab