Merge pull request #730 in VECTO/vecto-sim from...

Merge pull request #730 in VECTO/vecto-sim from ~EMQUARIMA/vecto-sim:feature/VECTO-873-add-digest-value-to-sumdata to develop

* commit 'f687d2ad':
  modify writing sum data to contain digest value
  adding testcase for digest value in sum-file

VectoCore/VectoCore/OutputData/FileIO/FileOutputWriter.cs

@@ -91,7 +91,7 @@ namespace TUGraz.VectoCore.OutputData.FileIO
 
 		public void WriteSumData(DataTable data)
 		{
-			VectoCSVFile.Write(SumFileName, data, true);
+			VectoCSVFile.Write(SumFileName, data, true, true);
 		}
 
 		public string GetModDataFileName(string runName, string cycleName, string runSuffix)

VectoCore/VectoCore/Utils/DataIntegrityHelper.cs

+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace TUGraz.VectoCore.Utils
+{
+	public class DataIntegrityHelper
+	{
+		public static string ComputeDigestValue(string[] lines)
+		{
+			var hash = System.Convert.ToBase64String(GetHash(string.Join("\n", lines)));
+
+			return string.Format("SHA256: {0}", hash);
+		}
+
+		public static byte[] GetHash(string inputString)
+		{
+			HashAlgorithm algorithm = SHA256.Create();  
+			return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString));
+		}
+	}
+}

VectoCore/VectoCore/Utils/VectoCSVFile.cs

@@ -31,6 +31,7 @@
 
 using Microsoft.VisualBasic.FileIO;
 using System;
+using System.Collections.Generic;
 using System.Data;
 using System.Globalization;
 using System.IO;
@@ -62,6 +63,7 @@ namespace TUGraz.VectoCore.Utils
 		private static readonly Regex HeaderFilter = new Regex(@"\[.*?\]|\<|\>", RegexOptions.Compiled);
 		private const string Delimiter = ",";
 		private const string Comment = "#";
+		private const string DigestValuePrefix = "#@";
 
 		/// <summary>
 		/// Reads a CSV file which is stored in Vecto-CSV-Format.
@@ -175,10 +177,11 @@ namespace TUGraz.VectoCore.Utils
 		/// <param name="fileName">Path to the file.</param>
 		/// <param name="table">The Datatable.</param>
 		/// <param name="addVersionHeader"></param>
-		public static void Write(string fileName, DataTable table, bool addVersionHeader = false)
+		/// <param name="addDigest"></param>
+		public static void Write(string fileName, DataTable table, bool addVersionHeader = false, bool addDigest = false)
 		{
 			using (var sw = new StreamWriter(new FileStream(fileName, FileMode.Create), Encoding.UTF8)) {
-				Write(sw, table, addVersionHeader);
+				Write(sw, table, addVersionHeader, addDigest);
 			}
 		}
 
@@ -190,21 +193,24 @@ namespace TUGraz.VectoCore.Utils
 		/// <param name="writer"></param>
 		/// <param name="table"></param>
 		/// <param name="addVersionHeader"></param>
-		public static void Write(StreamWriter writer, DataTable table, bool addVersionHeader = false)
+		/// <param name="addDigest"></param>
+		public static void Write(StreamWriter writer, DataTable table, bool addVersionHeader = false, bool addDigest = false)
 		{
 			if (writer == null) {
 				return;
 			}
+
+			var entries = new List<string>();
 			if (addVersionHeader) {
 				try {
-					writer.WriteLine("# VECTO {0} - {1}", VectoSimulationCore.VersionNumber,
-						DateTime.Now.ToString("dd.MM.yyyy HH:mm"));
+					entries.Add(string.Format("# VECTO {0} - {1}", VectoSimulationCore.VersionNumber,
+						DateTime.Now.ToString("dd.MM.yyyy HH:mm")));
 				} catch (Exception) {
-					writer.WriteLine("# VECTO {0} - {1}", "Unknown", DateTime.Now.ToString("dd.MM.yyyy HH:mm"));
+					entries.Add(string.Format("# VECTO {0} - {1}", "Unknown", DateTime.Now.ToString("dd.MM.yyyy HH:mm")));
 				}
 			}
 			var header = table.Columns.Cast<DataColumn>().Select(col => col.Caption ?? col.ColumnName);
-			writer.WriteLine(string.Join(Delimiter, header));
+			entries.Add(string.Join(Delimiter, header));
 
 			var columnFormatter = new Func<ConvertedSI, string>[table.Columns.Count];
 			for (var i = 0; i < table.Columns.Count; i++) {
@@ -221,21 +227,30 @@ namespace TUGraz.VectoCore.Utils
 				var formattedList = new string[items.Length];
 				for (var i = 0; i < items.Length; i++) {
 
-                    if (items[i] is SI) {
-                        formattedList[i] = columnFormatter[i]((SI)items[i]);
+					if (items[i] is SI) {
+						formattedList[i] = columnFormatter[i]((SI)items[i]);
 					} else if (items[i] is ConvertedSI) {
 						// todo mk-2017-10-02: maybe we also have to use decimals and showUnit from columnFormatter here?
 						formattedList[i] = columnFormatter[i]((ConvertedSI)items[i]);
 					} else {
-                        formattedList[i] = string.Format(CultureInfo.InvariantCulture, "{0}", items[i]);
-                    }
+						formattedList[i] = string.Format(CultureInfo.InvariantCulture, "{0}", items[i]);
+					}
 
-                    // if a string contains a "," then it has to be contained in quotes in order to be correctly recognized in a CSV file.
+					// if a string contains a "," then it has to be contained in quotes in order to be correctly recognized in a CSV file.
 					if (formattedList[i].Contains(Delimiter)) {
 						formattedList[i] = string.Format("\"{0}\"", formattedList[i]);
 					}
 				}
-				writer.WriteLine(string.Join(Delimiter, formattedList));
+				entries.Add(string.Join(Delimiter, formattedList));
+			}
+
+			if (addDigest) {
+				var digest = DataIntegrityHelper.ComputeDigestValue(entries.Where(x => !x.StartsWith(DigestValuePrefix)).ToArray());
+				entries.Add(string.Format("{0} {1}", DigestValuePrefix, digest));
+			}
+
+			foreach (var entry in entries) {
+				writer.WriteLine(entry);
 			}
 		}
 	}

VectoCore/VectoCore/VectoCore.csproj

@@ -222,6 +222,7 @@
     <Compile Include="OutputData\XML\XMLDeclarationWriter.cs" />
     <Compile Include="OutputData\XML\XMLEngineeringWriter.cs" />
     <Compile Include="OutputData\XML\XMLManufacturerReport.cs" />
+    <Compile Include="Utils\DataIntegrityHelper.cs" />
     <Compile Include="Utils\MeanShiftClustering.cs" />
     <Compile Include="Utils\ProviderExtensions.cs" />
     <Compile Include="Models\Declaration\AirDrag.cs" />

VectoCore/VectoCoreTest/Algorithms/CSVDigestValueTest.cs

+using System.Collections.Generic;
+using System.Data;
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+using NUnit.Framework.Internal;
+using TUGraz.VectoCommon.Utils;
+using TUGraz.VectoCore.Utils;
+
+namespace TUGraz.VectoCore.Tests.Algorithms
+{
+	[TestFixture]
+	public class CSVDigestValueTest
+	{
+
+		[TestCase]
+		public void TestDigestValueCreation()
+		{
+			var tbl = CreateDataTable(new[] { "t", "dt", "v" }, 5);
+
+			var str = new MemoryStream();
+			var writer = new StreamWriter(str);
+			VectoCSVFile.Write(writer, tbl, true, true);
+
+			writer.Flush();
+			str.Flush();
+			str.Seek(0, SeekOrigin.Begin);
+
+			var reader = new StreamReader(str);
+			var lines = new List<string>();
+			while (!reader.EndOfStream)
+				lines.Add(reader.ReadLine());
+
+			var last = lines.Last();
+
+			Assert.IsTrue(last.StartsWith("#@"), "Digest Identifier not found");
+			Assert.IsTrue(last.Contains("SHA256"), "Digest descriptor SHA256 not found");
+		}
+
+		[TestCase]
+		public void TestDigestValueValidation()
+		{
+			var tbl = CreateDataTable(new[] { "t", "dt", "v" }, 5);
+
+			var str = new MemoryStream();
+			var writer = new StreamWriter(str);
+			VectoCSVFile.Write(writer, tbl, true, true);
+
+			writer.Flush();
+			str.Flush();
+			str.Seek(0, SeekOrigin.Begin);
+
+			var reader = new StreamReader(str);
+			var lines = new List<string>();
+			while (!reader.EndOfStream)
+				lines.Add(reader.ReadLine());
+
+			var last = lines.Last();
+
+			Assert.IsTrue(last.StartsWith("#@"), "Digest Identifier not found");
+			Assert.IsTrue(last.Contains("SHA256"), "Digest descriptor SHA256 not found");
+
+			var otherLines = lines.Where(x => !x.StartsWith("#@")).ToArray();
+
+			var digest = DataIntegrityHelper.ComputeDigestValue(otherLines);
+
+			Assert.AreEqual(string.Format("#@ {0}", digest), last);
+		}
+
+		private static DataTable CreateDataTable(string[] cols, int numRows)
+		{
+			var tbl = new DataTable();
+			
+			foreach (var col in cols) {
+				tbl.Columns.Add(col, typeof(double));
+			}
+
+			var rnd = new Randomizer(873);
+			for (var i = 0; i < numRows; i++) {
+				var row = tbl.NewRow();
+				foreach (var col in cols) {
+					row[col] = rnd.NextDouble(100);
+				}
+
+				tbl.Rows.Add(row);
+			}
+
+			return tbl;
+		}
+
+
+		
+	}
+}

VectoCore/VectoCoreTest/VectoCoreTest.csproj

@@ -77,6 +77,7 @@
     <Otherwise />
   </Choose>
   <ItemGroup>
+    <Compile Include="Algorithms\CSVDigestValueTest.cs" />
     <Compile Include="Algorithms\MeanShiftClusteringTest.cs" />
     <Compile Include="Dummy\EngineFLDTest.cs" />
     <Compile Include="Exceptions\ExceptionTests.cs" />