CSV File: Robust CSV Reader (from VB.NET), Updated Tests for Aux Electric System

VectoCore/VectoCore/Models/Declaration/ElectricSystem.cs

@@ -56,14 +56,17 @@ namespace TUGraz.VectoCore.Models.Declaration
 			NormalizeTable(table);
 
 			foreach (DataRow row in table.Rows) {
-				var name = row.Field<string>("Technology");
-				foreach (MissionType mission in Enum.GetValues(typeof(MissionType))) {
-					Data[Tuple.Create(mission, name)] = row.ParseDouble(mission.ToString().ToLower()).SI<Watt>();
+				var name = row.Field<string>("technology");
+				foreach (DataColumn col in table.Columns) {
+					if (col.Caption != "technology") {
+						Data[Tuple.Create(col.Caption.ParseEnum<MissionType>(), name)] =
+							row.ParseDouble(col).SI<Watt>();
+					}
 				}
 			}
 		}
 
-		public override Watt Lookup(MissionType missionType, string technology)
+		public override Watt Lookup(MissionType missionType, string technology = "Standard technology")
 		{
 			var value = base.Lookup(missionType, technology);
 			return value / _alternator.Lookup(missionType);

VectoCore/VectoCore/Utils/StreamExtensions.cs

+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace TUGraz.VectoCore.Utils
+{
+	internal static class StreamExtensions
+	{
+		public static IEnumerable<string> ReadLines(this Stream stream)
+		{
+			using (var reader = new StreamReader(stream, Encoding.UTF8)) {
+				while (!reader.EndOfStream) {
+					yield return reader.ReadLine();
+				}
+			}
+		}
+	}
+}
\ No newline at end of file

VectoCore/VectoCore/Utils/VectoCSVFile.cs

@@ -37,6 +37,7 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
+using Microsoft.VisualBasic.FileIO;
 using TUGraz.VectoCommon.Exceptions;
 using TUGraz.VectoCommon.Models;
 using TUGraz.VectoCommon.Utils;
@@ -59,8 +60,8 @@ namespace TUGraz.VectoCore.Utils
 	public static class VectoCSVFile
 	{
 		private static readonly Regex HeaderFilter = new Regex(@"\[.*?\]|\<|\>", RegexOptions.Compiled);
-		private const char Delimiter = ',';
-		private const char Comment = '#';
+		private const string Delimiter = ",";
+		private const string Comment = "#";
 
 		/// <summary>
 		/// Reads a CSV file which is stored in Vecto-CSV-Format.
@@ -90,84 +91,59 @@ namespace TUGraz.VectoCore.Utils
 		/// <returns>A DataTable which represents the CSV File.</returns>
 		public static DataTable ReadStream(Stream stream, bool ignoreEmptyColumns = false, bool fullHeader = false)
 		{
-			try {
-				return ReadData(ReadLines(stream), ignoreEmptyColumns, fullHeader);
-			} catch (Exception e) {
-				LogManager.GetLogger(typeof(VectoCSVFile).FullName).Error(e);
-				throw new VectoException("Failed to read stream: " + e.Message, e);
-			}
-		}
+			var p = new TextFieldParser(stream) {
+				TextFieldType = FieldType.Delimited,
+				Delimiters = new[] { Delimiter },
+				CommentTokens = new[] { Comment },
+				HasFieldsEnclosedInQuotes = true,
+				TrimWhiteSpace = true
+			};
 
-		private static IEnumerable<string> ReadLines(Stream stream)
-		{
-			using (var reader = new StreamReader(stream, Encoding.UTF8)) {
-				while (!reader.EndOfStream) {
-					yield return reader.ReadLine();
-				}
-			}
-		}
+			string[] colsWithoutComment;
 
-		/// <summary>
-		/// 
-		/// </summary>
-		/// <param name="allLines"></param>
-		/// <param name="ignoreEmptyColumns"></param>
-		/// <param name="fullHeader"></param>
-		/// <returns></returns>
-		private static DataTable ReadData(IEnumerable<string> allLines, bool ignoreEmptyColumns = false,
-			bool fullHeader = false)
-		{
-			// trim, remove comments and filter empty lines
-			var lines = allLines
-				.Select(l => l.Trim())
-				.Select(l => l.Contains(Comment) ? l.Substring(0, l.IndexOf(Comment)) : l)
-				.Where(l => !string.IsNullOrWhiteSpace(l))
-				.GetEnumerator();
-
-			// start the enumerable
-			lines.MoveNext();
-
-			// add columns
-			var line = lines.Current;
-			if (!fullHeader) {
-				line = HeaderFilter.Replace(line, "");
+			try {
+				colsWithoutComment = p.ReadFields()
+					.Select(l => l.Contains(Comment) ? l.Substring(0, l.IndexOf(Comment)) : l)
+					.ToArray();
+			} catch (ArgumentNullException) {
+				throw new CSVReadException("CSV Read Error: File was empty.");
 			}
-			double tmp;
-			var splittedColumns = line
-				.Split(Delimiter);
 
-			var columns = splittedColumns
-				.Select(col => col.Trim())
+			double tmp;
+			var columns = colsWithoutComment
+				.Select(l => fullHeader ? l : HeaderFilter.Replace(l, ""))
+				.Select(l => l.Trim())
 				.Where(col => !double.TryParse(col, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp))
 				.ToList();
 
-			if (columns.Count > 0) {
-				// first line was a valid header: advance to first data line
-				lines.MoveNext();
-			} else {
+			var firstLineIsData = columns.Count == 0;
+
+			if (firstLineIsData) {
 				LogManager.GetLogger(typeof(VectoCSVFile).FullName)
 					.Warn("No valid Data Header found. Interpreting the first line as data line.");
 				// set the validColumns to: {"0", "1", "2", "3", ...} for all columns in first line.
-				columns = splittedColumns.Select((_, index) => index.ToString()).ToList();
+				columns = colsWithoutComment.Select((_, i) => i.ToString()).ToList();
 			}
 
 			var table = new DataTable();
 			foreach (var col in columns) {
 				table.Columns.Add(col);
 			}
-			if (lines.Current == null) {
+
+			if (p.EndOfData)
 				return table;
-			}
-			// read data into table
-			var i = 0;
-			do {
-				i++;
-				line = lines.Current;
 
-				var cells = line.Split(Delimiter).Select(s => s.Trim()).ToArray();
-				if (cells.Length != table.Columns.Count && !ignoreEmptyColumns) {
+			do {
+				var cells = firstLineIsData
+					? colsWithoutComment
+					: p.ReadFields()
+						.Select(l => l.Contains(Comment) ? l.Substring(0, l.IndexOf(Comment)) : l)
+						.Select(s => s.Trim())
+						.ToArray();
+				firstLineIsData = false;
+				if (table.Columns.Count != cells.Length && !ignoreEmptyColumns) {
 					throw new CSVReadException(
-						string.Format("Line {0}: The number of values is not correct. Expected {1} Columns, Got {2} Columns", i,
+						string.Format("Line {0}: The number of values is not correct. Expected {1} Columns, Got {2} Columns", p.LineNumber,
 							table.Columns.Count, cells.Length));
 				}
 
@@ -176,9 +152,10 @@ namespace TUGraz.VectoCore.Utils
 					table.Rows.Add(cells);
 				} catch (InvalidCastException e) {
 					throw new CSVReadException(
-						string.Format("Line {0}: The data format of a value is not correct. {1}", i, e.Message), e);
+						string.Format("Line {0}: The data format of a value is not correct. {1}", p.LineNumber, e.Message), e);
 				}
-			} while (lines.MoveNext());
+			} while (!p.EndOfData);
+
 			return table;
 		}
 

VectoCore/VectoCore/VectoCore.csproj

@@ -87,6 +87,7 @@
     <Reference Include="itextsharp">
       <HintPath>..\..\packages\iTextSharp.5.5.9\lib\itextsharp.dll</HintPath>
     </Reference>
+    <Reference Include="Microsoft.VisualBasic" />
     <Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\packages\Newtonsoft.Json.8.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
@@ -281,6 +282,7 @@
     <Compile Include="Models\Simulation\DataBus\IVehicleInfo.cs" />
     <Compile Include="Models\Simulation\IVehicleContainer.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Utils\StreamExtensions.cs" />
     <Compile Include="Utils\SwitchExtension.cs" />
     <Compile Include="Utils\VectoCSVFile.cs" />
     <Compile Include="Utils\DelaunayMap.cs" />

VectoCore/VectoCoreTest/FileIO/VectoCSVFileTest.cs

@@ -78,6 +78,45 @@ namespace TUGraz.VectoCore.Tests.FileIO
 			CollectionAssert.AreEqual(new[] { "4", "5", "6" }, table.Rows[1].ItemArray);
 		}
 
+		[Test]
+		public void VectoCSVFile_ReadStream_Escaped()
+		{
+			var stream = "a,b,c\n\"1,1\",2,3\n4,5,6".GetStream();
+			var table = VectoCSVFile.ReadStream(stream);
+
+			CollectionAssert.AreEqual(new[] { "a", "b", "c" }, table.Columns.Cast<DataColumn>().Select(c => c.ColumnName));
+			Assert.AreEqual(2, table.Rows.Count);
+
+			CollectionAssert.AreEqual(new[] { "1,1", "2", "3" }, table.Rows[0].ItemArray);
+			CollectionAssert.AreEqual(new[] { "4", "5", "6" }, table.Rows[1].ItemArray);
+		}
+
+		[Test]
+		public void VectoCSVFile_ReadStream_Comment()
+		{
+			var stream = "a,b,c\n\"1,1\",2,3#asdf\n4,5,6".GetStream();
+			var table = VectoCSVFile.ReadStream(stream);
+
+			CollectionAssert.AreEqual(new[] { "a", "b", "c" }, table.Columns.Cast<DataColumn>().Select(c => c.ColumnName));
+			Assert.AreEqual(2, table.Rows.Count);
+
+			CollectionAssert.AreEqual(new[] { "1,1", "2", "3" }, table.Rows[0].ItemArray);
+			CollectionAssert.AreEqual(new[] { "4", "5", "6" }, table.Rows[1].ItemArray);
+		}
+
+		[Test]
+		public void VectoCSVFile_ReadStream_EscapedComment()
+		{
+			var stream = "a,b,c\n\"1,1\",2,\"3#asdf\"\n4,5,6".GetStream();
+			var table = VectoCSVFile.ReadStream(stream);
+
+			CollectionAssert.AreEqual(new[] { "a", "b", "c" }, table.Columns.Cast<DataColumn>().Select(c => c.ColumnName));
+			Assert.AreEqual(2, table.Rows.Count);
+
+			CollectionAssert.AreEqual(new[] { "1,1", "2", "3" }, table.Rows[0].ItemArray);
+			CollectionAssert.AreEqual(new[] { "4", "5", "6" }, table.Rows[1].ItemArray);
+		}
+
 		[Test]
 		public void VectoCSVFile_ReadStream_No_Header()
 		{

VectoCore/VectoCoreTest/Models/Declaration/DeclarationDataTest.cs

@@ -44,7 +44,6 @@ using TUGraz.VectoCore.InputData.Reader.Impl;
 using TUGraz.VectoCore.Models.Declaration;
 using TUGraz.VectoCore.Models.SimulationComponent.Data;
 using TUGraz.VectoCore.Tests.Utils;
-using TUGraz.VectoCore.Utils;
 
 namespace TUGraz.VectoCore.Tests.Models.Declaration
 {
@@ -266,52 +265,30 @@ namespace TUGraz.VectoCore.Tests.Models.Declaration
 				torque.SI<NewtonMeter>() * Math.Pow((angularSpeed / referenceSpeed).Cast<Scalar>(), 2), torqueLookup);
 		}
 
-		[Test]
-		public void AuxElectricSystemTest()
+		[
+			TestCase(MissionType.LongHaul, "Standard technology", 1200, 0.7),
+			TestCase(MissionType.RegionalDelivery, "Standard technology", 1000, 0.7),
+			TestCase(MissionType.UrbanDelivery, "Standard technology", 1000, 0.7),
+			TestCase(MissionType.MunicipalUtility, "Standard technology", 1000, 0.7),
+			TestCase(MissionType.Construction, "Standard technology", 1000, 0.7),
+			TestCase(MissionType.LongHaul, "Standard technology - LED headlights, all", 1150, 0.7),
+			TestCase(MissionType.RegionalDelivery, "Standard technology - LED headlights, all", 950, 0.7),
+			TestCase(MissionType.UrbanDelivery, "Standard technology - LED headlights, all", 950, 0.7),
+			TestCase(MissionType.MunicipalUtility, "Standard technology - LED headlights, all", 950, 0.7),
+			TestCase(MissionType.Construction, "Standard technology - LED headlights, all", 950, 0.7),
+		]
+		public void AuxElectricSystemTest(MissionType mission, string technology, double value, double efficiency)
 		{
-			var es = DeclarationData.ElectricSystem;
-
-			var expected = new[] {
-				new { Mission = MissionType.LongHaul, Base = 1240.SI<Watt>(), LED = 1190.SI<Watt>(), Efficiency = 0.7 },
-				new {
-					Mission = MissionType.RegionalDelivery,
-					Base = 1055.SI<Watt>(),
-					LED = 1005.SI<Watt>(),
-					Efficiency = 0.7
-				},
-				new {
-					Mission = MissionType.UrbanDelivery,
-					Base = 974.SI<Watt>(),
-					LED = 924.SI<Watt>(),
-					Efficiency = 0.7
-				},
-				new {
-					Mission = MissionType.MunicipalUtility,
-					Base = 974.SI<Watt>(),
-					LED = 924.SI<Watt>(),
-					Efficiency = 0.7
-				},
-				new {
-					Mission = MissionType.Construction,
-					Base = 975.SI<Watt>(),
-					LED = 925.SI<Watt>(),
-					Efficiency = 0.7
-				},
-				new { Mission = MissionType.HeavyUrban, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 },
-				new { Mission = MissionType.Urban, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 },
-				new { Mission = MissionType.Suburban, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 },
-				new { Mission = MissionType.Interurban, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 },
-				new { Mission = MissionType.Coach, Base = 0.SI<Watt>(), LED = 0.SI<Watt>(), Efficiency = 1.0 }
-			};
-			Assert.AreEqual(expected.Length, Enum.GetValues(typeof(MissionType)).Length);
-
-			foreach (var expectation in expected) {
-				var baseConsumption = es.Lookup(expectation.Mission, null);
-				var leds = es.Lookup(expectation.Mission, "LED lights");
+			AssertHelper.AreRelativeEqual(value / efficiency, DeclarationData.ElectricSystem.Lookup(mission, technology));
+		}
 
-				AssertHelper.AreRelativeEqual(expectation.Base / expectation.Efficiency, baseConsumption);
-				AssertHelper.AreRelativeEqual(expectation.LED / expectation.Efficiency, leds);
-			}
+		[
+			TestCase(MissionType.Interurban, "Standard technology"),
+			TestCase(MissionType.LongHaul, "Standard technology - Flux-Compensator")
+		]
+		public void AuxElectricSystem_NotExistingError(MissionType mission, string technology)
+		{
+			AssertHelper.Exception<VectoException>(() => { DeclarationData.ElectricSystem.Lookup(mission, technology); });
 		}
 
 		[