Merge pull request #589 from krzychu124/fix-csv-translation-parser-with-tests

Fix CSV multi-line translation

Author: krzychu124

.gitattributes

@@ -61,3 +61,8 @@
 #*.PDF   diff=astextplain
 #*.rtf   diff=astextplain
 #*.RTF   diff=astextplain
+
+###############################################################################
+# Test files should not be normalized
+###############################################################################
+*.test binary

TLM/TLM/UI/Localization/LookupTable.cs

@@ -67,18 +67,17 @@ private void Load() {
             string filename = $"{Translation.RESOURCES_PREFIX}Translations.{Name}.csv";
             Log.Info($"Loading translations: {filename}");
 
-            string[] lines;
+            string firstLine;
+            string dataBlock;
             using (Stream st = Assembly.GetExecutingAssembly()
                                        .GetManifestResourceStream(filename)) {
                 using (var sr = new StreamReader(st, Encoding.UTF8)) {
-                    lines = sr.ReadToEnd()
-                              .Split(new[] { "\n", "\r\n" }, StringSplitOptions.None);
+                    ReadLines(sr, out firstLine, out dataBlock);
                 }
             }
 
             // Read each line as CSV line
             // Read language order in the first line
-            string firstLine = lines[0];
             var languageCodes = new List<string>();
             using (var sr = new StringReader(firstLine)) {
                 ReadCsvCell(sr); // skip
@@ -96,19 +95,38 @@ private void Load() {
                 }
             }
 
+            CollectTranslations(dataBlock, languageCodes, out AllLanguages);
+
+#if DUMP_TRANSLATIONS
+            DumpTranslationsToCsv();
+#endif
+            Log._Debug($"Loaded {AllLanguages.Count} different languages for {Name}");
+        }
+
+        /// <summary>
+        /// Collects translations to map - collection of keyValue pairs for each language code
+        /// </summary>
+        /// <param name="dataBlock">block of data (all rows excluding first)</param>
+        /// <param name="languageCodes">list of language codes</param>
+        /// <param name="allLanguages">result dictionary where all translation string will be collected</param>
+        private static void CollectTranslations(string dataBlock,
+                                                List<string> languageCodes,
+                                                out Dictionary<string, Dictionary<string, string>> allLanguages) {
+            allLanguages = new Dictionary<string, Dictionary<string, string>>();
             // Initialize empty dicts for each language
             foreach (string lang in languageCodes) {
-                AllLanguages[lang] = new Dictionary<string, string>();
+                allLanguages[lang] = new Dictionary<string, string>();
             }
 
             // first column is the translation key
             // Following columns are languages, following the order in AvailableLanguageCodes
-            foreach (string line in lines.Skip(1)) {
-                using (var sr = new StringReader(line)) {
+            using (var sr = new StringReader(dataBlock)) {
+                while (true) {
                     string key = ReadCsvCell(sr);
                     if (key.Length == 0) {
                         break; // last line is empty
                     }
+
                     foreach (string lang in languageCodes) {
                         string cell = ReadCsvCell(sr);
                         // Empty translations are not accepted for all languages other than English
@@ -117,15 +135,23 @@ private void Load() {
                             lang != Translation.DEFAULT_LANGUAGE_CODE) {
                             continue;
                         }
-                        AllLanguages[lang][key] = cell;
+
+                        allLanguages[lang][key] = cell;
                     }
                 }
             }
+        }
 
-#if DUMP_TRANSLATIONS
-            DumpTranslationsToCsv();
-#endif
-            Log._Debug($"Loaded {AllLanguages.Count} different languages for {Name}");
+        /// <summary>
+        /// Split stream of data on first row and remaining dataBlock
+        /// </summary>
+        /// <param name="sr">stream to read from</param>
+        /// <param name="firstLine">first line of tranlation - row with language code names</param>
+        /// <param name="dataBlock">string block of data (all remaining lines)</param>
+        /// <returns>collection of valid translation rows</returns>
+        private static void ReadLines(StreamReader sr, out string firstLine, out string dataBlock) {
+            firstLine = sr.ReadLine();
+            dataBlock = sr.ReadToEnd();
         }
 
         /// <summary>
@@ -162,18 +188,27 @@ private static string ReadCsvCell(StringReader sr) {
                             break;
                         }
                         case '\"': {
-                            // Found a '""', or a '",'
+                            // Found a '""', or a '",', or a '"/r', or a '"/r'
                             int peek = sr.Peek();
                             switch (peek) {
                                 case '\"': {
                                     sr.Read(); // consume the double quote
                                     sb.Append("\"");
                                     break;
                                 }
+                                case '\r':
+                                    //Followed by a \r then \n or just \n - end-of-string
+                                    sr.Read();// consume double quote
+                                    sr.Read();// consume \r
+                                    if (sr.Peek() == '\n') {
+                                        sr.Read(); // consume \n
+                                    }
+                                    return sb.ToString();
+                                case '\n':
                                 case ',':
                                 case -1: {
                                     // Followed by a comma or end-of-string
-                                    sr.Read(); // Consume the comma
+                                    sr.Read(); // Consume the comma or newLine(LF)
                                     return sb.ToString();
                                 }
                                 default: {
@@ -191,11 +226,15 @@ private static string ReadCsvCell(StringReader sr) {
                     }
                 }
             } else {
-                // Simple reading rules apply, read to the next comma or end-of-string
+                // Simple reading rules apply, read to the next comma, LF sequence or end-of-string
                 while (true) {
                     int next = sr.Read();
-                    if (next == -1 || next == ',') {
-                        break; // end-of-string or a comma
+                    if (next == -1 || next == ',' || next == '\n') {
+                        break; // end-of-string, a newLine or a comma
+                    }
+                    if (next == '\r' && sr.Peek() == '\n') {
+                        sr.Read(); //consume LF(\n) to complete CRLF(\r\n) newLine escape sequence
+                        break;
                     }
 
                     sb.Append((char)next, 1);

TLM/TMPE.UnitTest/TMPE.UnitTest.csproj

@@ -90,6 +90,7 @@
   </Choose>
   <ItemGroup>
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Translation\CsvParser.cs" />
     <Compile Include="Util\TinyDictionaryUnitTest.cs" />
     <Compile Include="Util\LogicUtilUnitTest.cs" />
   </ItemGroup>
@@ -111,6 +112,22 @@
     <Analyzer Include="..\packages\StyleCop.Analyzers.1.0.2\analyzers\dotnet\cs\StyleCop.Analyzers.CodeFixes.dll" />
     <Analyzer Include="..\packages\StyleCop.Analyzers.1.0.2\analyzers\dotnet\cs\StyleCop.Analyzers.dll" />
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="Translation\TestFiles\MultilineTestBlock_LF.test">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="Translation\TestFiles\MultilineTestBlock_CRLF.test">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="Translation\TestFiles\TestBlock_CRLF.test">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="Translation\TestFiles\TestBlock_LF.test">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <Choose>
     <When Condition="'$(VisualStudioVersion)' == '10.0' And '$(IsCodedUITest)' == 'True'">
       <ItemGroup>