XUnit – MemberDataAttribute and generic type inheritance

2017-06-19 TPodolakxunit

1. Introduction.

When I was writing Cake.Intellisense I was playing a lot with Roslyn syntax rewriters. The one thing which was pretty important for me at this time was a reliable set of tests which can quickly show if rewritten code is valid C# code and if it matches my expectations. As I don’t like to repeat my code I wanted to have a base test class which can be provided with different test cases from inherited classes. The initial implementation looked as follows

public abstract class CSharpSyntaxRewriterTest<T> where T : CSharpSyntaxRewriter
{
    public static IEnumerable<object[]> TestCases { get; set; }
    protected T Subject { get; }

    private readonly NSubstituteAutoMocker<T> _autoMock;
    
    protected CSharpSyntaxRewriterTest()
    {
        _autoMock = new NSubstituteAutoMocker<T>();
        Subject = _autoMock.ClassUnderTest;
    }

    [Theory]
    [MemberData(nameof(TestCases))]
    public void Verify(SyntaxRewriterTestCase testCase)
    {
        var inputTree = SyntaxFactory.ParseSyntaxTree(testCase.Input);
        var result = Subject.Visit(inputTree.GetRoot());

        inputTree.GetDiagnostics().Should().BeEmpty();
        result.GetDiagnostics().Should().BeEmpty();
        result.ToFullString().Should().Be(testCase.ExpectedResult);
    }
}

public abstract class CSharpSyntaxRewriterTest<T> where T : CSharpSyntaxRewriter

{

public static IEnumerable<object[]> TestCases { get; set; }

protected T Subject { get; }

private readonly NSubstituteAutoMocker<T> _autoMock;

protected CSharpSyntaxRewriterTest()

{

_autoMock = new NSubstituteAutoMocker<T>();

Subject = _autoMock.ClassUnderTest;

}

[Theory]

[MemberData(nameof(TestCases))]

public void Verify(SyntaxRewriterTestCase testCase)

{

var inputTree = SyntaxFactory.ParseSyntaxTree(testCase.Input);

var result = Subject.Visit(inputTree.GetRoot());

inputTree.GetDiagnostics().Should().BeEmpty();

result.GetDiagnostics().Should().BeEmpty();

result.ToFullString().Should().Be(testCase.ExpectedResult);

}

As you can see I have a generic base test class which holds test cases in a static field TestCases. The field is static as it is required by XUnit, this solution is perfectly valid because a static field is shared across all instances of the same type. Having a base test class ready I wanted to write a set of unit tests for following syntax rewriter

public class InternalToPublicClassModifierSyntaxRewriter : CSharpSyntaxRewriter
{ 
    public override SyntaxNode VisitClassDeclaration(ClassDeclarationSyntax node)
    {
        var internalModifierIndex = node.Modifiers.IndexOf(SyntaxKind.InternalKeyword);
            
        if (internalModifierIndex > -1)
        {
            var tokenInList = node.Modifiers[internalModifierIndex];
            var updatedModifiers = node.Modifiers.Replace(tokenInList,
                   SyntaxFactory.Token(SyntaxKind.PublicKeyword).WithLeadingTrivia(tokenInList.LeadingTrivia)
                        .WithTrailingTrivia(tokenInList.TrailingTrivia));

            node = node.WithModifiers(updatedModifiers);
         }

         return base.VisitClassDeclaration(node);
    }
}

public class InternalToPublicClassModifierSyntaxRewriter : CSharpSyntaxRewriter

{

public override SyntaxNode VisitClassDeclaration(ClassDeclarationSyntax node)

{

var internalModifierIndex = node.Modifiers.IndexOf(SyntaxKind.InternalKeyword);

if (internalModifierIndex > -1)

{

var tokenInList = node.Modifiers[internalModifierIndex];

var updatedModifiers = node.Modifiers.Replace(tokenInList,

SyntaxFactory.Token(SyntaxKind.PublicKeyword).WithLeadingTrivia(tokenInList.LeadingTrivia)

.WithTrailingTrivia(tokenInList.TrailingTrivia));

node = node.WithModifiers(updatedModifiers);

}

return base.VisitClassDeclaration(node);

}

The rewriter shown above is a very simple piece of code which replaces internal class modifier with a public one. Writing the tests for that class was supposed to be very easy, I just had to provide test cases with input code and expected result.

public class InternalToPublicClassModifierSyntaxRewriterTests
        : CSharpSyntaxRewriterTest<InternalToPublicClassModifierSyntaxRewriter>
{
    static InternalToPublicClassModifierSyntaxRewriterTests()
    {
        TestCases = new[]
        {
            new[] { ReplacesInternalModifierWithPublic() },
            new[] { LeavesModifiersIntactWhenClassModifierPublic() },
            new[] { LeavesModifiersIntactWhenClassModifierProtected() }
        };
    }

    private static SyntaxRewriterTestCase ReplacesInternalModifierWithPublic()
    {
        return new SyntaxRewriterTestCase(nameof(ReplacesInternalModifierWithPublic), 
                "internal class Foo { }", "public class Foo { }");
    }

    private static SyntaxRewriterTestCase LeavesModifiersIntactWhenClassModifierPublic()
    {
        return new SyntaxRewriterTestCase(nameof(LeavesModifiersIntactWhenClassModifierPublic),
                "public class Foo { }", "public class Foo { }");
    }

    private static SyntaxRewriterTestCase LeavesModifiersIntactWhenClassModifierProtected()
    {
        return new SyntaxRewriterTestCase(nameof(LeavesModifiersIntactWhenClassModifierProtected),
                "protected class Foo { }", "protected class Foo { }");
    }
}

public class InternalToPublicClassModifierSyntaxRewriterTests

: CSharpSyntaxRewriterTest<InternalToPublicClassModifierSyntaxRewriter>

{

static InternalToPublicClassModifierSyntaxRewriterTests()

{

TestCases = new[]

{

new[] { ReplacesInternalModifierWithPublic() },

new[] { LeavesModifiersIntactWhenClassModifierPublic() },

new[] { LeavesModifiersIntactWhenClassModifierProtected() }

};

}

private static SyntaxRewriterTestCase ReplacesInternalModifierWithPublic()

{

return new SyntaxRewriterTestCase(nameof(ReplacesInternalModifierWithPublic),

"internal class Foo { }", "public class Foo { }");

}

private static SyntaxRewriterTestCase LeavesModifiersIntactWhenClassModifierPublic()

{

return new SyntaxRewriterTestCase(nameof(LeavesModifiersIntactWhenClassModifierPublic),

"public class Foo { }", "public class Foo { }");

}

private static SyntaxRewriterTestCase LeavesModifiersIntactWhenClassModifierProtected()

{

return new SyntaxRewriterTestCase(nameof(LeavesModifiersIntactWhenClassModifierProtected),

"protected class Foo { }", "protected class Foo { }");

}

Unfortunately running the tests resulted in InvalidOperationException being thrown from XUnit

If you look closely at the message you will notice that the TestCases field used for providing data was not initialized, which means that the static constructor of class InternalToPublicClassModifierSyntaxRewriterTests was not run. That was pretty surprising for me that is why I wanted to investigated the issue a bit more.

2. Investigation.

I started from going back to the basics. As we probably all know, the static constructor is triggered when one of the following events occur:

An instance of the class type is created.
Any of the static members of the class type are referenced

The first condition is not met because instance of the test class is created after test cases are enumerated. That is why member data fields have to be a static fields/properties etc. When it comes to second condition the situation in here is a bit more complicated but if you take a look at the MemberDataAttributeBase class

public override IEnumerable<object[]> GetData(MethodInfo testMethod)
{
    Guard.ArgumentNotNull("testMethod", testMethod);

    var type = MemberType ?? testMethod.DeclaringType;
    var accessor = GetPropertyAccessor(type) ?? GetFieldAccessor(type) ?? GetMethodAccessor(type);
    // implementation omitted for brevity  
}

public override IEnumerable<object[]> GetData(MethodInfo testMethod)

{

Guard.ArgumentNotNull("testMethod", testMethod);

var type = MemberType ?? testMethod.DeclaringType;

var accessor = GetPropertyAccessor(type) ?? GetFieldAccessor(type) ?? GetMethodAccessor(type);

// implementation omitted for brevity

}

you will notice that member data looks for properties in DeclaringType of method. In my case declaring type is

XUnitMemberDataAndInheritance.CSharpSyntaxRewriterTest`1[[XUnitMemberDataAndInheritance.InternalToPublicClassModifierSyntaxRewriter]]

1	XUnitMemberDataAndInheritance.CSharpSyntaxRewriterTest`1[[XUnitMemberDataAndInheritance.InternalToPublicClassModifierSyntaxRewriter]]

not

InternalToPublicClassModifierSyntaxRewriterTests

1	InternalToPublicClassModifierSyntaxRewriterTests

So when XUnit accesses TestCase property the static constructor from InternalToPublicClassModifierSyntaxRewriterTests class will not be called either as the types does not match.

3. Solution.

In order to bend the behavior of XUnit to my needs, I had to ensure that the static constructor from inherited class is called. Fortunately, workaround is pretty straightforward – I had to introduce a custom MemberDataAttribute which calls static constructor from ReflectedType manually

[DataDiscoverer("Xunit.Sdk.MemberDataDiscoverer", "xunit.core")]
[AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
public class CustomMemberDataAttribute : MemberDataAttributeBase
{
    public CustomMemberDataAttribute(string memberName, params object[] parameters)
            : base(memberName, parameters)
    {
    }

    public override IEnumerable<object[]> GetData(MethodInfo testMethod)
    {
        var type = MemberType ?? testMethod.ReflectedType;
        System.Runtime.CompilerServices.RuntimeHelpers.RunClassConstructor(type.TypeHandle);
        return base.GetData(testMethod);
    }

    protected override object[] ConvertDataItem(MethodInfo testMethod, object item)
    {
        if (item == null)
        {
            return null;
        }

        var array = item as object[];
        if (array == null)
        {
            throw new ArgumentException(
                    $"Property {MemberName} on {MemberType ?? testMethod.ReflectedType} yielded an item that is not an object[]");
        }
            return array;
    }
}

[DataDiscoverer("Xunit.Sdk.MemberDataDiscoverer", "xunit.core")]

[AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]

public class CustomMemberDataAttribute : MemberDataAttributeBase

{

public CustomMemberDataAttribute(string memberName, params object[] parameters)

: base(memberName, parameters)

{

}

public override IEnumerable<object[]> GetData(MethodInfo testMethod)

{

var type = MemberType ?? testMethod.ReflectedType;

System.Runtime.CompilerServices.RuntimeHelpers.RunClassConstructor(type.TypeHandle);

return base.GetData(testMethod);

}

protected override object[] ConvertDataItem(MethodInfo testMethod, object item)

{

if (item == null)

{

return null;

}

var array = item as object[];

if (array == null)

{

throw new ArgumentException(

$"Property {MemberName} on {MemberType ?? testMethod.ReflectedType} yielded an item that is not an object[]");

}

return array;

}

Now all I have to do is use newly created attribute in CSharpSyntaxRewriterTest test class

[Theory]
[CustomMemberData(nameof(TestCases))]
public void Verify(SyntaxRewriterTestCase testCase)
{
    // implementation omitted for brevity        
}

[Theory]

[CustomMemberData(nameof(TestCases))]

public void Verify(SyntaxRewriterTestCase testCase)

{

// implementation omitted for brevity

}

From now on all of the test cases will be run correctly

Source code for this post can be found here

NSubstitute – returning value from IEnumerable

2017-06-122017-06-14 TPodolakNSubstitute

I’ve been using NSubstitute combined with automocks for quite some time already, but recently I’ve encountered quite interesting situation. Let’s assume that we have the following class we need to write tests for

public class PriceValidator : IValidator<Price>
{
    private readonly IEnumerable<IValidator<Price>> _innerValidators;

    public PriceValidator(IEnumerable<IValidator<Price>> innerValidators)
    {
         _innerValidators = innerValidators;
    }

    public bool IsValid(Price input)
    {
        return _innerValidators.Any() && _innerValidators.All(validator => validator.IsValid(input));
    }
}

public class PriceValidator : IValidator<Price>

{

private readonly IEnumerable<IValidator<Price>> _innerValidators;

public PriceValidator(IEnumerable<IValidator<Price>> innerValidators)

{

_innerValidators = innerValidators;

}

public bool IsValid(Price input)

{

return _innerValidators.Any() && _innerValidators.All(validator => validator.IsValid(input));

}

As mentioned before I wanted to test that class leveraging the concept of automocks. If you have never used them before, no worries, the idea is pretty simple. Instead of providing mock dependencies manually, you just generate them automatically. Once the dependencies are mocked you can retrieve and set them up if necessary. So in my case, I have to setup IEnumerable<IValidator<Price>> mock so that it yields some values. It can be achieved by mocking return values of GetEnumerator() method.

var automock = new NSubstituteAutoMocker<PriceValidator>();
var innerValidators = new List<IValidator<Price>>
{
    Substitute.For<IValidator<Price>>(),
};            
automock.Get<IEnumerable<IValidator<Price>>>().GetEnumerator().Returns(innerValidators.GetEnumerator());

var automock = new NSubstituteAutoMocker<PriceValidator>();

var innerValidators = new List<IValidator<Price>>

{

Substitute.For<IValidator<Price>>(),

};

automock.Get<IEnumerable<IValidator<Price>>>().GetEnumerator().Returns(innerValidators.GetEnumerator());

The entire test then can be written as

[Fact]
public void IsValid_ReturnsTrue_WhenAllInnerValidatorsAreValid()
{
    var automock = new NSubstituteAutoMocker<PriceValidator>();
    var innerValidators = new List<IValidator<Price>>
    {
        Substitute.For<IValidator<Price>>(),
    };
    innerValidators.ForEach(validator => validator.IsValid(Arg.Any<Price>()).Returns(true));
    automock.Get<IEnumerable<IValidator<Price>>>().GetEnumerator().Returns(innerValidators.GetEnumerator());
            
    var result = automock.ClassUnderTest.IsValid(new Price());

    innerValidators.ForEach(validator => validator.Received(1).IsValid(Arg.Any<Price>()));
    result.Should().BeTrue();
}

[Fact]

public void IsValid_ReturnsTrue_WhenAllInnerValidatorsAreValid()

{

var automock = new NSubstituteAutoMocker<PriceValidator>();

var innerValidators = new List<IValidator<Price>>

{

Substitute.For<IValidator<Price>>(),

};

innerValidators.ForEach(validator => validator.IsValid(Arg.Any<Price>()).Returns(true));

automock.Get<IEnumerable<IValidator<Price>>>().GetEnumerator().Returns(innerValidators.GetEnumerator());

var result = automock.ClassUnderTest.IsValid(new Price());

innerValidators.ForEach(validator => validator.Received(1).IsValid(Arg.Any<Price>()));

result.Should().BeTrue();

}

Everything looks good at that point, however if you run the tests, it will fail

That was quite surprising for me and I have to admit that at the beginning I thought it was a bug in NSubstitute.
However, if you looks closely at the implementation of the test, you will see that every time validator collection is enumerated, the same instance of the enumerator is used. As the enumerator keeps its state, the first enumeration (_innerValidators.Any()) will move a current element to the first position (which is the last position at the same time) so when I try to enumerate again (_innerValidators.All(validator => validator.IsValid(input)) enumeration will continue from the previous position and the iteration will end yielding no results. Fortunately, with NSubstitute you can configure the call so that it always returns a new value. The relevant code changes are shown below

automock.Get<IEnumerable<IValidator<Price>>>().GetEnumerator().Returns(callInfo => innerValidators.GetEnumerator());

1	automock.Get<IEnumerable<IValidator<Price>>>().GetEnumerator().Returns(callInfo => innerValidators.GetEnumerator());

Running the tests again will give us “green” result

Source code for this post can be found here

Dotnet CLI – running tests from multiple assemblies

2017-06-072017-06-09 TPodolakdotnet cli2 Comments

When you start looking for information how to run unit tests for .NET Core based projects in the command line, you will most probably stumble upon dotnet CLI and its dotnet test command.
The command works really great when you have just one test project in your solution, however, it doesn’t allow you to run tests from multiple projects at once. Of course, you can write little PowerShell which will run the command in the loop e.g.

Get-ChildItem | ? { $_.Name.Contains(".Tests.") } | ForEach-Object { Push-Location; Set-Location $_.Name; dotnet test; Pop-Location}

1	Get-ChildItem \| ? { $_.Name.Contains(".Tests.") } \| ForEach-Object { Push-Location; Set-Location $_.Name; dotnet test; Pop-Location}

Unfortunately, in this case, you won’t get an aggregated summary of tests results, instead, you will get a summary per test project.

Luckily dotnet CLI have another command for running tests – namely dotnet vstest. In this case, we do not operate on projects but we provide a location for assemblies with tests. So if you, for instance, would like to run unit and integration tests at once, you can write something like that

dotnet vstest .\DotnetCLIRunningTestsFromMultipleAssemblies.Tests.Integration\bin\Debug\netcoreapp1.1\DotnetCLIRunningTestsFromMultipleAssemblies.Tests.Integration.dll .\DotnetCLIRunningTestsFromMultipleAssemblies.Tests.Unit\bin\Debug\netcoreapp1.1\DotnetCLIRunningTestsFromMultipleAssemblies.Tests.Unit.dll

dotnet vstest .\DotnetCLIRunningTestsFromMultipleAssemblies.Tests.Integration\bin\Debug\netcoreapp1.1\DotnetCLIRunningTestsFromMultipleAssemblies.Tests.Integration.dll .\DotnetCLIRunningTestsFromMultipleAssemblies.Tests.Unit\bin\Debug\netcoreapp1.1\DotnetCLIRunningTestsFromMultipleAssemblies.Tests.Unit.dll

If you want a bit more flexible solution this simple PowerShell script will scan files and based on naming convention retrieve tests assemblies.

dotnet vstest (Get-ChildItem -recurse -File *.Tests.*dll | ? { $_.FullName -notmatch "\\obj\\?" })

1	dotnet vstest (Get-ChildItem -recurse -File .Tests.dll \| ? { $_.FullName -notmatch "\\obj\\?" })

One way or another, the dotnet vstest command will aggregate all of the test results and will present them as one summary.

One thing to note, dotnet vstest won’t build the solution, you have to run the build manually with dotnet build command.
Source code for this post can be found here

.NET BLOG

.NET, C# blog

Month: June 2017