NHibernate Cascade AllDeleteOrphan not working for unidirectional association

I'm having a difficult time setting up Fluent NHibernate mapping for my model.

I'm modelling a simple domain where a Forum has many Participants, each of them has many Tags.
Each of those relations is a unidirectional one to many composition

Because of that I would like to configure NHibernate to delete a Tag once the association between a Tag and Participant is removed (i.e. tag removed from collection) when I call ResetTags method

However, when I remove a Tag from collection of tags in Participant using ResetTags method I get the following exception when committing transaction

NHibernate.HibernateException HResult=0x80131500 Message=A collection with cascade="all-delete-orphan" was no longer referenced by the owning entity instance: LanguageWire.Dialogues.Domain.Dialogue.Participants Source=mscorlib

Please kindly let me know what I'm missing here.

Here is how the abbreviated code looks like

public class Forum : AggregateRoot
{
    private IList<Participant> _participants = new List<Participant>();
    public virtual IReadOnlyCollection<Participant> Participants => new ReadOnlyCollection<Participant>(_participants);
}

public class Participant : IEquatable<Participant>
{
    private int _id;
    private IList<Tag> _tags = new List<Tag>();

    public virtual void ResetTags(params Tag[] tags)
    {
        if (tags == null) throw new ArgumentNullException(nameof(tags));

        _tags.Clear();

        foreach (var tag in tags) _tags.Add(tag);
    }
}

public class Tag : IEquatable<Tag>
{
    private int _id;
    private byte _typeId;
    private int _entityId;
}

Here are my mappings

public class ForumMapping : ClassMap<Forum>
{
    public ForumMapping()
    {
        Id(x => x.Id);
        HasMany(x => x.Participants)
            .Not.Inverse()
            .Cascade.AllDeleteOrphan()
            .BatchSize(Default.BatchSize);
    }
}

public class ParticipantMapping : ClassMap<Participant>
{
    public ParticipantMapping()
    {
        Id(x => x.Id);
        HasMany(x => x.Tags)
            .Cascade.AllDeleteOrphan()
            .Not.Inverse()
            .Not.KeyNullable()
            .Not.KeyUpdate()
            .BatchSize(Default.BatchSize);

        Cache.ReadWrite().Region(nameof(Participant));
    }
}

public class TagMapping : ClassMap<Tag>
{
    public TagMapping()
    {
        Id(x => x.Id);
        Map(x => x.EntityId)
            .Not.Nullable()
            .Not.Update();
        Map(x => x.TypeId)
            .Not.Nullable()
            .Not.Update();
    }
}

By convention, mappings are to fields (not to properties, thus no problem with replacing NHibernate implementation collection with a new instance)

    public ISessionFactory BootstrapSessionFactory()
    {
        NHibernateLogger.SetLoggersFactory(new SerilogNHibernateLoggerFactory());
        return Fluently.Configure()
            .Database(MsSqlConfiguration.MsSql2012.ConnectionString(_dataAccessConfiguration.DatabaseConnectionString))
            .Cache(cache => cache.UseSecondLevelCache())
            .Mappings(m =>
                m.FluentMappings
                    .AddFromAssemblyOf<PersistenceSetup>()
                    .Conventions.Add(
                    DefaultAccess.CamelCaseField(CamelCasePrefix.Underscore),

Here is how I save data

        _session.FlushMode = FlushMode.Commit;
        using (var transaction = _session.BeginTransaction())
        {
            foreach (var aggregate in _aggregates)
                await _session.SaveOrUpdateAsync(aggregate, cancellationToken);

            await transaction.CommitAsync(cancellationToken);
        }

The versions of packages I use:

<PackageReference Include="FluentNHibernate" Version="2.1.2" />
<PackageReference Include="NHibernate" Version="5.1.3" />