Skip to content

Commit 1ac6d84

Browse files
dwcullopJakenVeina
authored andcommitted
Correct sample standard deviation calculations
The sample deviation divisor was applied outside the square root while the item count divided inside it, so every StdDev overload returned a value smaller than the true sample standard deviation. This changes returned numeric results. The corrected value is the previous value multiplied by sqrt(n-1), so results are unchanged for two items and diverge upward as the item count grows. The int and long overloads additionally no longer truncate the intermediate mean-square term before the subtraction, so integer inputs with a fractional mean now report the correct fractional variance. Those cross products and the resulting numerator are exact, provided the accumulators have not overflowed; the final division and square root still round. Restores the StdDev test coverage that was lost, with 21 cases spanning all five numeric overloads. Backport of b8e09ea from main.
1 parent 75e4452 commit 1ac6d84

2 files changed

Lines changed: 321 additions & 10 deletions

File tree

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
// Copyright (c) 2011-2026 Roland Pheasant. All rights reserved.
2+
// Roland Pheasant licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for full license information.
4+
5+
using System;
6+
using System.Linq;
7+
using System.Reactive.Linq;
8+
9+
using Bogus;
10+
11+
using DynamicData.Aggregation;
12+
using DynamicData.Tests.Utilities;
13+
14+
using FluentAssertions;
15+
16+
using Xunit;
17+
using Xunit.Abstractions;
18+
19+
namespace DynamicData.Tests.AggregationTests;
20+
21+
/// <summary>Verifies sample standard deviation for the supported numeric overloads.</summary>
22+
public sealed class StdDevFixture
23+
{
24+
private const int Seed = 0x2409_1071;
25+
26+
private readonly Randomizer _randomizer = new(Seed);
27+
28+
public StdDevFixture(ITestOutputHelper output)
29+
=> output.WriteLine($"{nameof(StdDevFixture)} seed: {Seed:X8}");
30+
31+
/// <summary>Verifies that cache standard deviation applies the sample divisor to the variance.</summary>
32+
[Theory]
33+
[InlineData(nameof(Int32))]
34+
[InlineData(nameof(Int64))]
35+
[InlineData(nameof(Single))]
36+
[InlineData(nameof(Double))]
37+
[InlineData(nameof(Decimal))]
38+
public void Cache_ThreeEquallySpacedValues_ReportsTheirSpacing(string numericType)
39+
{
40+
// Arrange
41+
var center = _randomizer.Int(-100, 100);
42+
var spacing = _randomizer.Int(1, 30);
43+
var values = new[] { center - spacing, center, center + spacing };
44+
var fallback = _randomizer.Int(1, 100);
45+
using var source = new TestSourceCache<int, int>(static value => value);
46+
var standardDeviation = numericType switch
47+
{
48+
nameof(Int32) => source.Connect().StdDev(static value => value, fallback),
49+
nameof(Int64) => source.Connect().StdDev(static value => (long)value, fallback),
50+
nameof(Single) => source.Connect().StdDev(static value => (float)value, fallback),
51+
nameof(Double) => source.Connect().StdDev(static value => (double)value, fallback),
52+
nameof(Decimal) => source.Connect().StdDev(static value => (decimal)value, fallback).Select(static value => (double)value),
53+
_ => throw new ArgumentOutOfRangeException(nameof(numericType))
54+
};
55+
using var subscription = standardDeviation
56+
.ValidateSynchronization()
57+
.RecordValues(out var results);
58+
59+
// Act
60+
source.AddOrUpdate(values);
61+
62+
// Assert
63+
results.Error.Should().BeNull(because: "all values are within the supported numeric ranges");
64+
results.RecordedValues.Should().ContainSingle(because: "one source edit produces one aggregate")
65+
.Which.Should().Be(spacing, because: "the sample variance of three equally spaced values is the square of their spacing");
66+
}
67+
68+
/// <summary>Verifies that list standard deviation applies the sample divisor to the variance.</summary>
69+
[Theory]
70+
[InlineData(nameof(Int32))]
71+
[InlineData(nameof(Int64))]
72+
[InlineData(nameof(Single))]
73+
[InlineData(nameof(Double))]
74+
[InlineData(nameof(Decimal))]
75+
public void List_ThreeEquallySpacedValues_ReportsTheirSpacing(string numericType)
76+
{
77+
// Arrange
78+
var center = _randomizer.Int(-100, 100);
79+
var spacing = _randomizer.Int(1, 30);
80+
var values = new[] { center - spacing, center, center + spacing };
81+
var fallback = _randomizer.Int(1, 100);
82+
using var source = new TestSourceList<int>();
83+
var standardDeviation = numericType switch
84+
{
85+
nameof(Int32) => source.Connect().StdDev(static value => value, fallback),
86+
nameof(Int64) => source.Connect().StdDev(static value => (long)value, fallback),
87+
nameof(Single) => source.Connect().StdDev(static value => (float)value, fallback),
88+
nameof(Double) => source.Connect().StdDev(static value => (double)value, fallback),
89+
nameof(Decimal) => source.Connect().StdDev(static value => (decimal)value, fallback).Select(static value => (double)value),
90+
_ => throw new ArgumentOutOfRangeException(nameof(numericType))
91+
};
92+
using var subscription = standardDeviation
93+
.ValidateSynchronization()
94+
.RecordValues(out var results);
95+
96+
// Act
97+
source.AddRange(values);
98+
99+
// Assert
100+
results.Error.Should().BeNull(because: "all values are within the supported numeric ranges");
101+
results.RecordedValues.Should().ContainSingle(because: "one source edit produces one aggregate")
102+
.Which.Should().Be(spacing, because: "the sample variance of three equally spaced values is the square of their spacing");
103+
}
104+
105+
/// <summary>Verifies that integer overloads retain the fractional part of the mean when calculating variance.</summary>
106+
[Theory]
107+
[InlineData(nameof(Int32))]
108+
[InlineData(nameof(Int64))]
109+
public void IntegerValues_FractionalMean_PreservesFractionalVariance(string numericType)
110+
{
111+
// Arrange
112+
var start = _randomizer.Int(-100, 100);
113+
var spacing = _randomizer.Int(2, 30);
114+
var adjustment = _randomizer.Int(1, 2);
115+
var values = new[] { start, start + spacing, start + spacing + spacing + adjustment };
116+
var mean = values.Average();
117+
var expected = Math.Sqrt(values.Sum(value => Math.Pow(value - mean, 2)) / (values.Length - 1));
118+
var fallback = _randomizer.Int(1, 100);
119+
using var source = new TestSourceCache<int, int>(static value => value);
120+
var standardDeviation = numericType switch
121+
{
122+
nameof(Int32) => source.Connect().StdDev(static value => value, fallback),
123+
nameof(Int64) => source.Connect().StdDev(static value => (long)value, fallback),
124+
_ => throw new ArgumentOutOfRangeException(nameof(numericType))
125+
};
126+
using var subscription = standardDeviation
127+
.ValidateSynchronization()
128+
.RecordValues(out var results);
129+
130+
// Act
131+
source.AddOrUpdate(values);
132+
133+
// Assert
134+
results.Error.Should().BeNull(because: "integer inputs with a fractional mean are valid");
135+
results.RecordedValues.Should().ContainSingle(because: "one source edit produces one aggregate")
136+
.Which.Should().BeApproximately(expected, 1e-10, because: "integer inputs do not imply an integer-valued mean or variance");
137+
}
138+
139+
/// <summary>Verifies that integer overloads keep the variance of values offset far from zero, where the raw moments exceed the exactly representable integer range of <see cref="double"/>.</summary>
140+
[Theory]
141+
[InlineData(nameof(Int32))]
142+
[InlineData(nameof(Int64))]
143+
public void IntegerValues_OffsetBeyondExactDoubleRange_ReportsTheirSpacing(string numericType)
144+
{
145+
// Arrange
146+
var offset = _randomizer.Int(100_000_000, 500_000_000);
147+
var spacing = _randomizer.Int(1, 30);
148+
var values = new[] { offset, offset + spacing, offset + spacing + spacing };
149+
var fallback = _randomizer.Int(1, 100);
150+
using var source = new TestSourceCache<int, int>(static value => value);
151+
var standardDeviation = numericType switch
152+
{
153+
nameof(Int32) => source.Connect().StdDev(static value => value, fallback),
154+
nameof(Int64) => source.Connect().StdDev(static value => (long)value, fallback),
155+
_ => throw new ArgumentOutOfRangeException(nameof(numericType))
156+
};
157+
using var subscription = standardDeviation
158+
.ValidateSynchronization()
159+
.RecordValues(out var results);
160+
161+
// Act
162+
source.AddOrUpdate(values);
163+
164+
// Assert
165+
results.Error.Should().BeNull(because: "the squared moments of these values remain within the accumulator range");
166+
results.RecordedValues.Should().ContainSingle(because: "one source edit produces one aggregate")
167+
.Which.Should().Be(spacing, because: "standard deviation is translation invariant, so offsetting the values cannot change their spread");
168+
}
169+
170+
/// <summary>Verifies that integer overloads keep the variance of far-offset values across removals and replacements, not only on the initial population.</summary>
171+
[Theory]
172+
[InlineData(nameof(Int32))]
173+
[InlineData(nameof(Int64))]
174+
public void IntegerValues_OffsetBeyondExactDoubleRangeThenRemovedAndReplaced_TracksTheirSpacing(string numericType)
175+
{
176+
// Arrange
177+
var offset = _randomizer.Int(100_000_000, 500_000_000);
178+
var spacing = _randomizer.Int(1, 30);
179+
var values = new[] { offset, offset + spacing, offset + spacing + spacing, offset + spacing + spacing + spacing };
180+
var replacement = offset - spacing;
181+
var afterRemoval = new[] { values[0], values[1], values[2] };
182+
var afterReplacement = new[] { replacement, values[1], values[2] };
183+
var fallback = _randomizer.Int(1, 100);
184+
using var source = new TestSourceList<int>();
185+
var standardDeviation = numericType switch
186+
{
187+
nameof(Int32) => source.Connect().StdDev(static value => value, fallback),
188+
nameof(Int64) => source.Connect().StdDev(static value => (long)value, fallback),
189+
_ => throw new ArgumentOutOfRangeException(nameof(numericType))
190+
};
191+
using var subscription = standardDeviation
192+
.ValidateSynchronization()
193+
.RecordValues(out var results);
194+
195+
// Act
196+
source.AddRange(values);
197+
source.Remove(values[3]);
198+
source.Replace(values[0], replacement);
199+
200+
// Assert
201+
results.Error.Should().BeNull(because: "the squared moments of these values remain within the accumulator range");
202+
results.RecordedValues.Should().HaveCount(3, because: "each of the three source edits produces one aggregate");
203+
results.RecordedValues[0].Should().BeApproximately(SampleStandardDeviation(values), 1e-9, because: "the initial population is four equally spaced values");
204+
results.RecordedValues[1].Should().BeApproximately(SampleStandardDeviation(afterRemoval), 1e-9, because: "removing an item must recalculate the variance from the remaining offset values");
205+
results.RecordedValues[2].Should().BeApproximately(SampleStandardDeviation(afterReplacement), 1e-9, because: "replacing an item must recalculate the variance from the remaining offset values");
206+
}
207+
208+
/// <summary>Verifies that clearing the input returns the configured fallback rather than evaluating an undefined variance.</summary>
209+
[Theory]
210+
[InlineData(nameof(Int32))]
211+
[InlineData(nameof(Int64))]
212+
[InlineData(nameof(Single))]
213+
[InlineData(nameof(Double))]
214+
[InlineData(nameof(Decimal))]
215+
public void Cache_AllItemsRemoved_ReportsFallback(string numericType)
216+
{
217+
// Arrange
218+
var center = _randomizer.Int(-100, 100);
219+
var spacing = _randomizer.Int(1, 30);
220+
var values = new[] { center - spacing, center, center + spacing };
221+
var fallback = _randomizer.Int(1, 100);
222+
using var source = new TestSourceCache<int, int>(static value => value);
223+
source.AddOrUpdate(values);
224+
var standardDeviation = numericType switch
225+
{
226+
nameof(Int32) => source.Connect().StdDev(static value => value, fallback),
227+
nameof(Int64) => source.Connect().StdDev(static value => (long)value, fallback),
228+
nameof(Single) => source.Connect().StdDev(static value => (float)value, fallback),
229+
nameof(Double) => source.Connect().StdDev(static value => (double)value, fallback),
230+
nameof(Decimal) => source.Connect().StdDev(static value => (decimal)value, fallback).Select(static value => (double)value),
231+
_ => throw new ArgumentOutOfRangeException(nameof(numericType))
232+
};
233+
using var subscription = standardDeviation
234+
.ValidateSynchronization()
235+
.RecordValues(out var results);
236+
237+
// Act
238+
source.Clear();
239+
240+
// Assert
241+
results.Error.Should().BeNull(because: "an empty collection has a defined fallback");
242+
results.RecordedValues[^1].Should().Be(fallback, because: "sample variance is not calculated for fewer than two items");
243+
}
244+
245+
/// <summary>Calculates the reference sample standard deviation, centring the values so the reference itself is not subject to the cancellation under test.</summary>
246+
private static double SampleStandardDeviation(params int[] values)
247+
{
248+
var offset = values.Min();
249+
var centred = values.Select(value => (double)(value - offset)).ToArray();
250+
var mean = centred.Average();
251+
return Math.Sqrt(centred.Sum(value => (value - mean) * (value - mean)) / (centred.Length - 1));
252+
}
253+
}

0 commit comments

Comments
 (0)
⌂ 换网址