xunit

1. Introduction.

When I was writing Cake.Intellisense I was playing a lot with Roslyn syntax rewriters. The one thing which was pretty important for me at this time was a reliable set of tests which can quickly show if rewritten code is valid C# code and if it matches my expectations. As I don’t like to repeat my code I wanted to have a base test class which can be provided with different test cases from inherited classes. The initial implementation looked as follows

public abstract class CSharpSyntaxRewriterTest<T> where T : CSharpSyntaxRewriter
{
    public static IEnumerable<object[]> TestCases { get; set; }
    protected T Subject { get; }

    private readonly NSubstituteAutoMocker<T> _autoMock;
    
    protected CSharpSyntaxRewriterTest()
    {
        _autoMock = new NSubstituteAutoMocker<T>();
        Subject = _autoMock.ClassUnderTest;
    }

    [Theory]
    [MemberData(nameof(TestCases))]
    public void Verify(SyntaxRewriterTestCase testCase)
    {
        var inputTree = SyntaxFactory.ParseSyntaxTree(testCase.Input);
        var result = Subject.Visit(inputTree.GetRoot());

        inputTree.GetDiagnostics().Should().BeEmpty();
        result.GetDiagnostics().Should().BeEmpty();
        result.ToFullString().Should().Be(testCase.ExpectedResult);
    }
}

public abstract class CSharpSyntaxRewriterTest<T> where T : CSharpSyntaxRewriter

{

public static IEnumerable<object[]> TestCases { get; set; }

protected T Subject { get; }

private readonly NSubstituteAutoMocker<T> _autoMock;

protected CSharpSyntaxRewriterTest()

{

_autoMock = new NSubstituteAutoMocker<T>();

Subject = _autoMock.ClassUnderTest;

}

[Theory]

[MemberData(nameof(TestCases))]

public void Verify(SyntaxRewriterTestCase testCase)

{

var inputTree = SyntaxFactory.ParseSyntaxTree(testCase.Input);

var result = Subject.Visit(inputTree.GetRoot());

inputTree.GetDiagnostics().Should().BeEmpty();

result.GetDiagnostics().Should().BeEmpty();

result.ToFullString().Should().Be(testCase.ExpectedResult);

}

As you can see I have a generic base test class which holds test cases in a static field TestCases. The field is static as it is required by XUnit, this solution is perfectly valid because a static field is shared across all instances of the same type. Having a base test class ready I wanted to write a set of unit tests for following syntax rewriter

public class InternalToPublicClassModifierSyntaxRewriter : CSharpSyntaxRewriter
{ 
    public override SyntaxNode VisitClassDeclaration(ClassDeclarationSyntax node)
    {
        var internalModifierIndex = node.Modifiers.IndexOf(SyntaxKind.InternalKeyword);
            
        if (internalModifierIndex > -1)
        {
            var tokenInList = node.Modifiers[internalModifierIndex];
            var updatedModifiers = node.Modifiers.Replace(tokenInList,
                   SyntaxFactory.Token(SyntaxKind.PublicKeyword).WithLeadingTrivia(tokenInList.LeadingTrivia)
                        .WithTrailingTrivia(tokenInList.TrailingTrivia));

            node = node.WithModifiers(updatedModifiers);
         }

         return base.VisitClassDeclaration(node);
    }
}

public class InternalToPublicClassModifierSyntaxRewriter : CSharpSyntaxRewriter

{

public override SyntaxNode VisitClassDeclaration(ClassDeclarationSyntax node)

{

var internalModifierIndex = node.Modifiers.IndexOf(SyntaxKind.InternalKeyword);

if (internalModifierIndex > -1)

{

var tokenInList = node.Modifiers[internalModifierIndex];

var updatedModifiers = node.Modifiers.Replace(tokenInList,

SyntaxFactory.Token(SyntaxKind.PublicKeyword).WithLeadingTrivia(tokenInList.LeadingTrivia)

.WithTrailingTrivia(tokenInList.TrailingTrivia));

node = node.WithModifiers(updatedModifiers);

}

return base.VisitClassDeclaration(node);

}

The rewriter shown above is a very simple piece of code which replaces internal class modifier with a public one. Writing the tests for that class was supposed to be very easy, I just had to provide test cases with input code and expected result.

public class InternalToPublicClassModifierSyntaxRewriterTests
        : CSharpSyntaxRewriterTest<InternalToPublicClassModifierSyntaxRewriter>
{
    static InternalToPublicClassModifierSyntaxRewriterTests()
    {
        TestCases = new[]
        {
            new[] { ReplacesInternalModifierWithPublic() },
            new[] { LeavesModifiersIntactWhenClassModifierPublic() },
            new[] { LeavesModifiersIntactWhenClassModifierProtected() }
        };
    }

    private static SyntaxRewriterTestCase ReplacesInternalModifierWithPublic()
    {
        return new SyntaxRewriterTestCase(nameof(ReplacesInternalModifierWithPublic), 
                "internal class Foo { }", "public class Foo { }");
    }

    private static SyntaxRewriterTestCase LeavesModifiersIntactWhenClassModifierPublic()
    {
        return new SyntaxRewriterTestCase(nameof(LeavesModifiersIntactWhenClassModifierPublic),
                "public class Foo { }", "public class Foo { }");
    }

    private static SyntaxRewriterTestCase LeavesModifiersIntactWhenClassModifierProtected()
    {
        return new SyntaxRewriterTestCase(nameof(LeavesModifiersIntactWhenClassModifierProtected),
                "protected class Foo { }", "protected class Foo { }");
    }
}

public class InternalToPublicClassModifierSyntaxRewriterTests

: CSharpSyntaxRewriterTest<InternalToPublicClassModifierSyntaxRewriter>

{

static InternalToPublicClassModifierSyntaxRewriterTests()

{

TestCases = new[]

{

new[] { ReplacesInternalModifierWithPublic() },

new[] { LeavesModifiersIntactWhenClassModifierPublic() },

new[] { LeavesModifiersIntactWhenClassModifierProtected() }

};

}

private static SyntaxRewriterTestCase ReplacesInternalModifierWithPublic()

{

return new SyntaxRewriterTestCase(nameof(ReplacesInternalModifierWithPublic),

"internal class Foo { }", "public class Foo { }");

}

private static SyntaxRewriterTestCase LeavesModifiersIntactWhenClassModifierPublic()

{

return new SyntaxRewriterTestCase(nameof(LeavesModifiersIntactWhenClassModifierPublic),

"public class Foo { }", "public class Foo { }");

}

private static SyntaxRewriterTestCase LeavesModifiersIntactWhenClassModifierProtected()

{

return new SyntaxRewriterTestCase(nameof(LeavesModifiersIntactWhenClassModifierProtected),

"protected class Foo { }", "protected class Foo { }");

}

Unfortunately running the tests resulted in InvalidOperationException being thrown from XUnit

If you look closely at the message you will notice that the TestCases field used for providing data was not initialized, which means that the static constructor of class InternalToPublicClassModifierSyntaxRewriterTests was not run. That was pretty surprising for me that is why I wanted to investigated the issue a bit more.

2. Investigation.

I started from going back to the basics. As we probably all know, the static constructor is triggered when one of the following events occur:

An instance of the class type is created.
Any of the static members of the class type are referenced

The first condition is not met because instance of the test class is created after test cases are enumerated. That is why member data fields have to be a static fields/properties etc. When it comes to second condition the situation in here is a bit more complicated but if you take a look at the MemberDataAttributeBase class

public override IEnumerable<object[]> GetData(MethodInfo testMethod)
{
    Guard.ArgumentNotNull("testMethod", testMethod);

    var type = MemberType ?? testMethod.DeclaringType;
    var accessor = GetPropertyAccessor(type) ?? GetFieldAccessor(type) ?? GetMethodAccessor(type);
    // implementation omitted for brevity  
}

public override IEnumerable<object[]> GetData(MethodInfo testMethod)

{

Guard.ArgumentNotNull("testMethod", testMethod);

var type = MemberType ?? testMethod.DeclaringType;

var accessor = GetPropertyAccessor(type) ?? GetFieldAccessor(type) ?? GetMethodAccessor(type);

// implementation omitted for brevity

}

you will notice that member data looks for properties in DeclaringType of method. In my case declaring type is

XUnitMemberDataAndInheritance.CSharpSyntaxRewriterTest`1[[XUnitMemberDataAndInheritance.InternalToPublicClassModifierSyntaxRewriter]]

1	XUnitMemberDataAndInheritance.CSharpSyntaxRewriterTest`1[[XUnitMemberDataAndInheritance.InternalToPublicClassModifierSyntaxRewriter]]

not

InternalToPublicClassModifierSyntaxRewriterTests

1	InternalToPublicClassModifierSyntaxRewriterTests

So when XUnit accesses TestCase property the static constructor from InternalToPublicClassModifierSyntaxRewriterTests class will not be called either as the types does not match.

3. Solution.

In order to bend the behavior of XUnit to my needs, I had to ensure that the static constructor from inherited class is called. Fortunately, workaround is pretty straightforward – I had to introduce a custom MemberDataAttribute which calls static constructor from ReflectedType manually

[DataDiscoverer("Xunit.Sdk.MemberDataDiscoverer", "xunit.core")]
[AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
public class CustomMemberDataAttribute : MemberDataAttributeBase
{
    public CustomMemberDataAttribute(string memberName, params object[] parameters)
            : base(memberName, parameters)
    {
    }

    public override IEnumerable<object[]> GetData(MethodInfo testMethod)
    {
        var type = MemberType ?? testMethod.ReflectedType;
        System.Runtime.CompilerServices.RuntimeHelpers.RunClassConstructor(type.TypeHandle);
        return base.GetData(testMethod);
    }

    protected override object[] ConvertDataItem(MethodInfo testMethod, object item)
    {
        if (item == null)
        {
            return null;
        }

        var array = item as object[];
        if (array == null)
        {
            throw new ArgumentException(
                    $"Property {MemberName} on {MemberType ?? testMethod.ReflectedType} yielded an item that is not an object[]");
        }
            return array;
    }
}

[DataDiscoverer("Xunit.Sdk.MemberDataDiscoverer", "xunit.core")]

[AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]

public class CustomMemberDataAttribute : MemberDataAttributeBase

{

public CustomMemberDataAttribute(string memberName, params object[] parameters)

: base(memberName, parameters)

{

}

public override IEnumerable<object[]> GetData(MethodInfo testMethod)

{

var type = MemberType ?? testMethod.ReflectedType;

System.Runtime.CompilerServices.RuntimeHelpers.RunClassConstructor(type.TypeHandle);

return base.GetData(testMethod);

}

protected override object[] ConvertDataItem(MethodInfo testMethod, object item)

{

if (item == null)

{

return null;

}

var array = item as object[];

if (array == null)

{

throw new ArgumentException(

$"Property {MemberName} on {MemberType ?? testMethod.ReflectedType} yielded an item that is not an object[]");

}

return array;

}

Now all I have to do is use newly created attribute in CSharpSyntaxRewriterTest test class

[Theory]
[CustomMemberData(nameof(TestCases))]
public void Verify(SyntaxRewriterTestCase testCase)
{
    // implementation omitted for brevity        
}

[Theory]

[CustomMemberData(nameof(TestCases))]

public void Verify(SyntaxRewriterTestCase testCase)

{

// implementation omitted for brevity

}

From now on all of the test cases will be run correctly

Source code for this post can be found here

1. Introduction

XUnit provides two ways of sharing data between tests – ICollectionFixture and IClassFixture. The first one allows you to shared context across collections and the second one across test classes. I’ve already had a couple of cases in which these fixtures were not enough. Basically, I wanted to share data between all tests in given assembly – without worrying in which test class or test collection given test is. In these circumstances, I’ve usually used some kind of old fashioned singleton or custom TestFrameworkExecutor. I’ve never liked those solutions, fortunately recently I’ve come across a nice little library – xunit.assemblyfixture.

2. Usage

Xunit.assemblyfixture allows you to share data between tests in given assembly via IAssemblyFixture interface. The usage is basically the same as in other XUnit’s fixtures. All we have to do is to create a class which instance we want to share with other tests

public class TestFixture : IDisposable
{
    public static int InitializationCounter { get; private set; }

    public TestFixture()
    {
        InitializationCounter++;
    }

    public void Dispose()
    {
        // free resources if necessary
    }
}

public class TestFixture : IDisposable

{

public static int InitializationCounter { get; private set; }

public TestFixture()

{

InitializationCounter++;

}

public void Dispose()

{

// free resources if necessary

}

And then create test classes which implement IAssemblyFixture interface. If we want to have access to the fixture from the tests, we can inject instance of TFixture via constructor

public class TestBase : IAssemblyFixture<TestFixture>
{
    protected readonly ITestOutputHelper TestOutputHelper;
    protected readonly TestFixture Fixture;

    protected TestBase(ITestOutputHelper testOutputHelper, TestFixture fixture)
    {
        this.TestOutputHelper = testOutputHelper;
        this.Fixture = fixture;
    }

    protected void DisplayInitializationCount()
    {
        TestOutputHelper.WriteLine($"Initialization count: {TestFixture.InitializationCounter}");
    }
}

public class TestBase : IAssemblyFixture<TestFixture>

{

protected readonly ITestOutputHelper TestOutputHelper;

protected readonly TestFixture Fixture;

protected TestBase(ITestOutputHelper testOutputHelper, TestFixture fixture)

{

this.TestOutputHelper = testOutputHelper;

this.Fixture = fixture;

}

protected void DisplayInitializationCount()

{

TestOutputHelper.WriteLine($"Initialization count: {TestFixture.InitializationCounter}");

}

Xunit.assemblyfixture fixture will ensure that given test class is shared via all the tests and is initialized only once.

If you want to share multiple classes across the assembly, you can, of course, use IAssemblyFixture multiple times.

3. Visual Studio 2017 – XUnit beta tests runners

As for now VS 2017 RC requires beta runners of XUnit in order to run our unit tests. xunit.assemblyfixture seems not to cooperate with them smoothly. However, there is a simple workaround to fix that. All we have to do is to add the following attribute to our test project

[assembly: TestFramework("Xunit.TestFramework", "xunit.assemblyfixture")]

1	[assembly: TestFramework("Xunit.TestFramework", "xunit.assemblyfixture")]

And from now on, XUnit correctly can inject assembly fixtures into our test classes

Source code for this post can be found here

.NET BLOG

.NET, C# blog

XUnit – MemberDataAttribute and generic type inheritance