process tags : extra-careful normalization

tracer/src/Datadog.Trace/ProcessTags.cs

@@ -10,7 +10,7 @@
 using System.IO;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Processors;
-using Datadog.Trace.Util;
+using Datadog.Trace.SourceGenerators;
 
 namespace Datadog.Trace;
 
@@ -42,15 +42,33 @@ private static List<string> GetTagsList()
     /// </summary>
     private static void AddNormalizedTag(this List<string> tags, string key, string? value)
     {
-        if (string.IsNullOrEmpty(value))
+        if (StringUtil.IsNullOrWhiteSpace(value))
         {
             return;
         }
 
-        // TraceUtil.NormalizeTag does almost exactly what we want, except it allows ':',
-        // which we don't want because we use it as a key/value separator.
-        var normalizedValue = TraceUtil.NormalizeTag(value).Replace(oldChar: ':', newChar: '_');
-        tags.Add($"{key}:{normalizedValue}");
+        var normalizedValue = NormalizeTagValue(value);
+        // check length because normalization can squish the string to nothing
+        if (normalizedValue.Length > 0)
+        {
+            tags.Add($"{key}:{normalizedValue}");
+        }
+    }
+
+    [TestingAndPrivateOnly]
+    internal static string NormalizeTagValue(string tagValue)
+    {
+        // TraceUtil.NormalizeTag does almost exactly what we want, except it allows ':', which we don't want because we use it as a key/value separator.
+        // We need to replace ':' before calling NormalizeTag because there is a logic to remove duplicate underscores.
+        var normalized = TraceUtil.NormalizeTag(tagValue.Replace(oldChar: ':', newChar: '_'));
+
+        // truncate to 100 char, which the max allowed for a service name, and this is the only usage for those tags
+        if (normalized.Length > 100)
+        {
+            return normalized.Substring(startIndex: 0, length: 100);
+        }
+
+        return normalized;
     }
 
     private static string GetSerializedTagsFromList(List<string> tags)

tracer/test/Datadog.Trace.Tests/ProcessTagsTests.cs

@@ -30,4 +30,61 @@ public void TagsPresentWhenEnabled()
             });
         // cannot really assert on content because it depends on how the tests are run.
     }
+
+    [Theory]
+    [InlineData("#test_starting_hash", "test_starting_hash")]
+    [InlineData("TestCAPSandSuch", "testcapsandsuch")]
+    [InlineData("Test Conversion Of Weird !@#$%^&**() Characters", "test_conversion_of_weird_characters")]
+    [InlineData("$#weird_starting", "weird_starting")]
+    [InlineData("disallowed:c0l0ns", "disallowed_c0l0ns")]
+    [InlineData("1love", "love")]
+    [InlineData("123456", "")]
+    [InlineData("7.0", "")] // this is not ideal
+    [InlineData("ünicöde", "ünicöde")]
+    [InlineData("ünicöde:metäl", "ünicöde_metäl")]
+    [InlineData("Data🐨dog🐶 繋がっ⛰てて", "data_dog_繋がっ_てて")]
+    [InlineData(" spaces   ", "spaces")]
+    [InlineData(" #hashtag!@#spaces #__<>#  ", "hashtag_spaces")]
+    [InlineData(":testing", "testing")]
+    [InlineData("_foo", "foo")]
+    [InlineData(":::test", "test")]
+    [InlineData("contiguous_____underscores", "contiguous_underscores")]
+    [InlineData("foo_", "foo")]
+    [InlineData("", "")]
+    [InlineData(" ", "")]
+    [InlineData("ok", "ok")]
+    [InlineData("AlsO:ök", "also_ök")]
+    [InlineData(":still_ok", "still_ok")]
+    [InlineData("___trim", "trim")]
+    [InlineData("fun:ky__tag/1", "fun_ky_tag/1")]
+    [InlineData("fun:ky@tag/2", "fun_ky_tag/2")]
+    [InlineData("fun:ky@@@tag/3", "fun_ky_tag/3")]
+    [InlineData("tag:1/2.3", "tag_1/2.3")]
+    [InlineData("---fun:k####y_ta@#g/1_@@#", "fun_k_y_ta_g/1")]
+    [InlineData("AlsO:œ#@ö))œk", "also_œ_ö_œk")]
+    [InlineData("test\x99\x008faaa", "test_aaa")]
+    [InlineData("test\x99\x8f", "test")]
+    [InlineData(" regulartag ", "regulartag")]
+    [InlineData("\u017Fodd_\u017Fcase\u017F", "\u017Fodd_\u017Fcase\u017F")]
+    [InlineData("™Ö™Ö™™Ö™", "ö_ö_ö")]
+    [InlineData("a�", "a")]
+    [InlineData("a��", "a")]
+    [InlineData("a��b", "a_b")]
+    public void TestNormalization(string tagValue, string expectedValue)
+    {
+        ProcessTags.NormalizeTagValue(tagValue).Should().Be(expectedValue);
+    }
+
+    [Fact]
+    public void TestNormalizationsTruncation()
+    {
+        // cannot write those as `Theory` because the parameters need to be constant values
+        var tagValue = new string(c: 'a', count: 888);
+        var expected = new string(c: 'a', count: 100);
+        ProcessTags.NormalizeTagValue(tagValue).Should().Be(expected);
+
+        tagValue = "a" + new string(c: '➰', count: 799);
+        expected = "a";
+        ProcessTags.NormalizeTagValue(tagValue).Should().Be(expected);
+    }
 }