Where is the flaw in my attempt to implement a JsonConverter that handles interfaces and abstract classes?

Question

I am trying to create an implementation of JsonConverter that can deserialize interfaces and abstract classes by using the first derived type that can be deserialized through default deserialization. Here is my unit test to give you can idea:

    [TestMethod]
    public void BaseTypeConverterCanConvertPolymorphicClass()
    {
        var polymorphicClass = new ConcreteFoo()
        {
            SomeString = "test string",
            Bar = new ConcreteBar()
            {
                SomeInt = 18237
            }
        };

        string serialized = JsonConvert.SerializeObject(polymorphicClass);

        IFoo deserialized = JsonConvert.DeserializeObject<IFoo>(serialized);

        Assert.IsTrue(deserialized is ConcreteFoo);
    }

    [JsonConverter(typeof(BaseTypeConverter))]
    private interface IFoo
    {
        [JsonProperty("someString")]
        string SomeString { get; }

        [JsonProperty("bar")]
        IBar Bar { get; }
    }

    private class ConcreteFoo : IFoo
    {
        public string SomeString { get; set; }

        public IBar Bar { get; set; }
    }

    [JsonConverter(typeof(BaseTypeConverter))]
    private interface IBar
    {
        [JsonProperty("someInt")]
        int SomeInt { get; }
    }

    private class ConcreteBar : IBar
    {
        public int SomeInt { get; set; }
    }

    private class OtherConcreteBar : IBar
    {
        public int SomeInt { get; set; }

        [JsonProperty("someDouble")]
        public double SomeDouble { get; set; }
    }

and here is my implementation:

public class BaseTypeConverter : JsonConverter
{
    private static IDictionary<Type, ICollection<Type>> cache = new Dictionary<Type, ICollection<Type>>();

    public override bool CanConvert(Type objectType)
    {
        return objectType.IsAbstract || objectType.IsInterface;
    }

    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
    {
        IEnumerable<Type> derived = GetDerivedTypes(objectType);

        var defaultSerializer = JsonSerializer.CreateDefault();

        foreach (Type type in derived)
        {
            object deserialized = defaultSerializer.Deserialize(reader, type);

            if (deserialized != null)
            {
                return deserialized;
            }
        }

        throw new JsonException($"Could not deserialize type {objectType} into any of the dervied types: {string.Join(",", derived)}.");
    }

    public override bool CanWrite { get { return false; } }

    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
    {
        throw new NotImplementedException($"Should never have to write because {nameof(CanWrite)} is {CanWrite}.");
    }

    private static IEnumerable<Type> GetDerivedTypes(Type baseType)
    {
        if (cache.ContainsKey(baseType))
        {
            return cache[baseType];
        }

        var derivedTypes =
            (from domainAssembly in AppDomain.CurrentDomain.GetAssemblies()
             from assemblyType in domainAssembly.GetTypes()
             where baseType.IsAssignableFrom(assemblyType)
                && baseType != assemblyType
             select assemblyType).ToList();

        cache[baseType] = derivedTypes;

        return derivedTypes;
    }
}

The problem I'm finding is that somehow defaultSerializer.Deserialize(reader, type); is re-calling my ReadJson method rather than the "default" one like I expect. In other words, it calls it on type ConcreteFoo.

Where is the flaw in my logic?