Driver taints kernel and can't be unloaded

I have built a kernel module which is causing some problem. Some time after loading it, the driver stops working and at that moment, when I try to use rmmod to unload it, it blocks and I have to manually restart my device. I have managed to get this trace by using the main console of my card with a USB-TTL connexion:

Probe: Successful
xeno_sprinte_ida on AUART3: physical address=0x80070000 membase=0xc8e3c000 irq=234 uartclk=24000000
Init: Successful
------------[ cut here ]------------
------------[ cut here ]------------
WARNING: CPU: 0 PID: 825 at drivers/clk/clk.c:130 clk_enable_lock+0x98/0xd4
Modules linked in: moduleXenoUartSimple(O-) snd_soc_mxs_sgtl5000 snd_soc_sgtl5000 snd_soc_mxs snd_soc_mxs_pcm snd_soc_core snd_pcm_dmaengine snd_pcm snd_timer snd soundcore ums_alauda uas ums_onetouch ums_cypress ums_karma ums_jumpshot ums_eneub6250 firmware_class ums_usbat ums_sddr55 ums_sddr09 ums_freecom ums_datafab ums_isd200 ums_realtek usb_storage scsi_mod ci_hdrc_usb2 ci_hdrc_imx usbmisc_imx ci_hdrc ehci_hcd usbmon usbcore usb_common mxs_mmc sdhci_pltfm sdhci mmc_block mmc_core jffs2 ntfs ext2 vfat fat ipv6 lm75 hwmon fuse xeno_udd xeno_rtipc xeno_can_flexcan xeno_can [last unloaded: moduleXenoUartSimple]
CPU: 0 PID: 825 Comm: rmmod Tainted: G           O    4.9.24-ipipe-SPOS-0.2 #1
Hardware name: Freescale MXS (Device Tree)
I-pipe domain: Linux
[<c000f540>] (unwind_backtrace) from [<c000d5d0>] (show_stack+0x10/0x14)
[<c000d5d0>] (show_stack) from [<c0018cb0>] (__warn+0xd4/0xfc)
[<c0018cb0>] (__warn) from [<c0018d88>] (warn_slowpath_null+0x20/0x28)
[<c0018d88>] (warn_slowpath_null) from [<c0250dfc>] (clk_enable_lock+0x98/0xd4)
[<c0250dfc>] (clk_enable_lock) from [<c0252500>] (clk_core_enable_lock+0xc/0x2c)
[<c0252500>] (clk_core_enable_lock) from [<c0282ac0>] (pl011_console_write+0x20/0x1ac)
[<c0282ac0>] (pl011_console_write) from [<c00480f0>] (call_console_drivers.constprop.11+0x98/0xac)
[<c00480f0>] (call_console_drivers.constprop.11) from [<c0049104>] (console_unlock+0x2d0/0x6f0)
[<c0049104>] (console_unlock) from [<c00497e4>] (vprintk_emit+0x2c0/0x49c)
[<c00497e4>] (vprintk_emit) from [<c0049b4c>] (vprintk_default+0x20/0x28)
[<c0049b4c>] (vprintk_default) from [<c00ade9c>] (printk+0xe8/0x188)
[<c00ade9c>] (printk) from [<c0018c04>] (__warn+0x28/0xfc)
[<c0018c04>] (__warn) from [<c0018d88>] (warn_slowpath_null+0x20/0x28)
[<c0018d88>] (warn_slowpath_null) from [<c0252464>] (clk_core_disable_lock+0x18/0x24)
[<c0252464>] (clk_core_disable_lock) from [<bf2fc184>] (rt_mxs_auart_remove+0x1c/0x54 [moduleXenoUartSimple])
[<bf2fc184>] (rt_mxs_auart_remove [moduleXenoUartSimple]) from [<c028f620>] (__device_release_driver+0x70/0xfc)
[<c028f620>] (__device_release_driver) from [<c028feb8>] (driver_detach+0xfc/0x100)
[<c028feb8>] (driver_detach) from [<c028f1f4>] (bus_remove_driver+0x4c/0xa0)
[<c028f1f4>] (bus_remove_driver) from [<bf2fcdd8>] (rt_mxs_auart_exit+0xc/0x20 [moduleXenoUartSimple])
[<bf2fcdd8>] (rt_mxs_auart_exit [moduleXenoUartSimple]) from [<c006fd5c>] (SyS_delete_module+0x15c/0x1cc)
[<c006fd5c>] (SyS_delete_module) from [<c000a500>] (ret_fast_syscall+0x0/0x38)
---[ end trace 94d65edb72d57359 ]---
------------[ cut here ]------------
WARNING: CPU: 0 PID: 825 at drivers/clk/clk.c:131 clk_enable_lock+0xc0/0xd4
Modules linked in: moduleXenoUartSimple(O-) snd_soc_mxs_sgtl5000 snd_soc_sgtl5000 snd_soc_mxs snd_soc_mxs_pcm snd_soc_core snd_pcm_dmaengine snd_pcm snd_timer snd soundcore ums_alauda uas ums_onetouch ums_cypress ums_karma ums_jumpshot ums_eneub6250 firmware_class ums_usbat ums_sddr55 ums_sddr09 ums_freecom ums_datafab ums_isd200 ums_realtek usb_storage scsi_mod ci_hdrc_usb2 ci_hdrc_imx usbmisc_imx ci_hdrc ehci_hcd usbmon usbcore usb_common mxs_mmc sdhci_pltfm sdhci mmc_block mmc_core jffs2 ntfs ext2 vfat fat ipv6 lm75 hwmon fuse xeno_udd xeno_rtipc xeno_can_flexcan xeno_can [last unloaded: moduleXenoUartSimple]
CPU: 0 PID: 825 Comm: rmmod Tainted: G        W  O    4.9.24-ipipe-SPOS-0.2 #1
Hardware name: Freescale MXS (Device Tree)
I-pipe domain: Linux
[<c000f540>] (unwind_backtrace) from [<c000d5d0>] (show_stack+0x10/0x14)
[<c000d5d0>] (show_stack) from [<c0018cb0>] (__warn+0xd4/0xfc)
[<c0018cb0>] (__warn) from [<c0018d88>] (warn_slowpath_null+0x20/0x28)
[<c0018d88>] (warn_slowpath_null) from [<c0250e24>] (clk_enable_lock+0xc0/0xd4)
[<c0250e24>] (clk_enable_lock) from [<c0252500>] (clk_core_enable_lock+0xc/0x2c)
[<c0252500>] (clk_core_enable_lock) from [<c0282ac0>] (pl011_console_write+0x20/0x1ac)
[<c0282ac0>] (pl011_console_write) from [<c00480f0>] (call_console_drivers.constprop.11+0x98/0xac)
[<c00480f0>] (call_console_drivers.constprop.11) from [<c0049104>] (console_unlock+0x2d0/0x6f0)
[<c0049104>] (console_unlock) from [<c00497e4>] (vprintk_emit+0x2c0/0x49c)
[<c00497e4>] (vprintk_emit) from [<c0049b4c>] (vprintk_default+0x20/0x28)
[<c0049b4c>] (vprintk_default) from [<c00ade9c>] (printk+0xe8/0x188)
[<c00ade9c>] (printk) from [<c0018c04>] (__warn+0x28/0xfc)
[<c0018c04>] (__warn) from [<c0018d88>] (warn_slowpath_null+0x20/0x28)
[<c0018d88>] (warn_slowpath_null) from [<c0252464>] (clk_core_disable_lock+0x18/0x24)
[<c0252464>] (clk_core_disable_lock) from [<bf2fc184>] (rt_mxs_auart_remove+0x1c/0x54 [moduleXenoUartSimple])
[<bf2fc184>] (rt_mxs_auart_remove [moduleXenoUartSimple]) from [<c028f620>] (__device_release_driver+0x70/0xfc)
[<c028f620>] (__device_release_driver) from [<c028feb8>] (driver_detach+0xfc/0x100)
[<c028feb8>] (driver_detach) from [<c028f1f4>] (bus_remove_driver+0x4c/0xa0)
[<c028f1f4>] (bus_remove_driver) from [<bf2fcdd8>] (rt_mxs_auart_exit+0xc/0x20 [moduleXenoUartSimple])
[<bf2fcdd8>] (rt_mxs_auart_exit [moduleXenoUartSimple]) from [<c006fd5c>] (SyS_delete_module+0x15c/0x1cc)
[<c006fd5c>] (SyS_delete_module) from [<c000a500>] (ret_fast_syscall+0x0/0x38)
---[ end trace 94d65edb72d5735a ]---
WARNING: CPU: 0 PID: 825 at drivers/clk/clk.c:594 clk_core_disable_lock+0x18/0x24
Modules linked in: moduleXenoUartSimple(O-) snd_soc_mxs_sgtl5000 snd_soc_sgtl5000 snd_soc_mxs snd_soc_mxs_pcm snd_soc_core snd_pcm_dmaengine snd_pcm snd_timer snd soundcore ums_alauda uas ums_onetouch ums_cypress ums_karma ums_jumpshot ums_eneub6250 firmware_class ums_usbat ums_sddr55 ums_sddr09 ums_freecom ums_datafab ums_isd200 ums_realtek usb_storage scsi_mod ci_hdrc_usb2 ci_hdrc_imx usbmisc_imx ci_hdrc ehci_hcd usbmon usbcore usb_common mxs_mmc sdhci_pltfm sdhci mmc_block mmc_core jffs2 ntfs ext2 vfat fat ipv6 lm75 hwmon fuse xeno_udd xeno_rtipc xeno_can_flexcan xeno_can [last unloaded: moduleXenoUartSimple]
CPU: 0 PID: 825 Comm: rmmod Tainted: G        W  O    4.9.24-ipipe-SPOS-0.2 #1
Hardware name: Freescale MXS (Device Tree)
I-pipe domain: Linux
[<c000f540>] (unwind_backtrace) from [<c000d5d0>] (show_stack+0x10/0x14)
[<c000d5d0>] (show_stack) from [<c0018cb0>] (__warn+0xd4/0xfc)
[<c0018cb0>] (__warn) from [<c0018d88>] (warn_slowpath_null+0x20/0x28)
[<c0018d88>] (warn_slowpath_null) from [<c0252464>] (clk_core_disable_lock+0x18/0x24)
[<c0252464>] (clk_core_disable_lock) from [<bf2fc184>] (rt_mxs_auart_remove+0x1c/0x54 [moduleXenoUartSimple])
[<bf2fc184>] (rt_mxs_auart_remove [moduleXenoUartSimple]) from [<c028f620>] (__device_release_driver+0x70/0xfc)
[<c028f620>] (__device_release_driver) from [<c028feb8>] (driver_detach+0xfc/0x100)
[<c028feb8>] (driver_detach) from [<c028f1f4>] (bus_remove_driver+0x4c/0xa0)
[<c028f1f4>] (bus_remove_driver) from [<bf2fcdd8>] (rt_mxs_auart_exit+0xc/0x20 [moduleXenoUartSimple])
[<bf2fcdd8>] (rt_mxs_auart_exit [moduleXenoUartSimple]) from [<c006fd5c>] (SyS_delete_module+0x15c/0x1cc)
[<c006fd5c>] (SyS_delete_module) from [<c000a500>] (ret_fast_syscall+0x0/0x38)
---[ end trace 94d65edb72d5735b ]---
------------[ cut here ]------------
WARNING: CPU: 0 PID: 825 at drivers/clk/clk.c:140 clk_enable_unlock+0xb0/0xf8
Modules linked in: moduleXenoUartSimple(O-) snd_soc_mxs_sgtl5000 snd_soc_sgtl5000 snd_soc_mxs snd_soc_mxs_pcm snd_soc_core snd_pcm_dmaengine snd_pcm snd_timer snd soundcore ums_alauda uas ums_onetouch ums_cypress ums_karma ums_jumpshot ums_eneub6250 firmware_class ums_usbat ums_sddr55 ums_sddr09 ums_freecom ums_datafab ums_isd200 ums_realtek usb_storage scsi_mod ci_hdrc_usb2 ci_hdrc_imx usbmisc_imx ci_hdrc ehci_hcd usbmon usbcore usb_common mxs_mmc sdhci_pltfm sdhci mmc_block mmc_core jffs2 ntfs ext2 vfat fat ipv6 lm75 hwmon fuse xeno_udd xeno_rtipc xeno_can_flexcan xeno_can [last unloaded: moduleXenoUartSimple]
CPU: 0 PID: 825 Comm: rmmod Tainted: G        W  O    4.9.24-ipipe-SPOS-0.2 #1
Hardware name: Freescale MXS (Device Tree)
I-pipe domain: Linux
[<c000f540>] (unwind_backtrace) from [<c000d5d0>] (show_stack+0x10/0x14)
[<c000d5d0>] (show_stack) from [<c0018cb0>] (__warn+0xd4/0xfc)
[<c0018cb0>] (__warn) from [<c0018d88>] (warn_slowpath_null+0x20/0x28)
[<c0018d88>] (warn_slowpath_null) from [<c0252404>] (clk_enable_unlock+0xb0/0xf8)
[<c0252404>] (clk_enable_unlock) from [<bf2fc184>] (rt_mxs_auart_remove+0x1c/0x54 [moduleXenoUartSimple])
[<bf2fc184>] (rt_mxs_auart_remove [moduleXenoUartSimple]) from [<c028f620>] (__device_release_driver+0x70/0xfc)
[<c028f620>] (__device_release_driver) from [<c028feb8>] (driver_detach+0xfc/0x100)
[<c028feb8>] (driver_detach) from [<c028f1f4>] (bus_remove_driver+0x4c/0xa0)
[<c028f1f4>] (bus_remove_driver) from [<bf2fcdd8>] (rt_mxs_auart_exit+0xc/0x20 [moduleXenoUartSimple])
[<bf2fcdd8>] (rt_mxs_auart_exit [moduleXenoUartSimple]) from [<c006fd5c>] (SyS_delete_module+0x15c/0x1cc)
[<c006fd5c>] (SyS_delete_module) from [<c000a500>] (ret_fast_syscall+0x0/0x38)
---[ end trace 94d65edb72d5735c ]---
------------[ cut here ]------------
WARNING: CPU: 0 PID: 825 at drivers/clk/clk.c:141 clk_enable_unlock+0xe4/0xf8
Modules linked in: moduleXenoUartSimple(O-) snd_soc_mxs_sgtl5000 snd_soc_sgtl5000 snd_soc_mxs snd_soc_mxs_pcm snd_soc_core snd_pcm_dmaengine snd_pcm snd_timer snd soundcore ums_alauda uas ums_onetouch ums_cypress ums_karma ums_jumpshot ums_eneub6250 firmware_class ums_usbat ums_sddr55 ums_sddr09 ums_freecom ums_datafab ums_isd200 ums_realtek usb_storage scsi_mod ci_hdrc_usb2 ci_hdrc_imx usbmisc_imx ci_hdrc ehci_hcd usbmon usbcore usb_common mxs_mmc sdhci_pltfm sdhci mmc_block mmc_core jffs2 ntfs ext2 vfat fat ipv6 lm75 hwmon fuse xeno_udd xeno_rtipc xeno_can_flexcan xeno_can [last unloaded: moduleXenoUartSimple]
CPU: 0 PID: 825 Comm: rmmod Tainted: G        W  O    4.9.24-ipipe-SPOS-0.2 #1
Hardware name: Freescale MXS (Device Tree)
I-pipe domain: Linux
[<c000f540>] (unwind_backtrace) from [<c000d5d0>] (show_stack+0x10/0x14)
[<c000d5d0>] (show_stack) from [<c0018cb0>] (__warn+0xd4/0xfc)
[<c0018cb0>] (__warn) from [<c0018d88>] (warn_slowpath_null+0x20/0x28)
[<c0018d88>] (warn_slowpath_null) from [<c0252438>] (clk_enable_unlock+0xe4/0xf8)
[<c0252438>] (clk_enable_unlock) from [<bf2fc184>] (rt_mxs_auart_remove+0x1c/0x54 [moduleXenoUartSimple])
[<bf2fc184>] (rt_mxs_auart_remove [moduleXenoUartSimple]) from [<c028f620>] (__device_release_driver+0x70/0xfc)
[<c028f620>] (__device_release_driver) from [<c028feb8>] (driver_detach+0xfc/0x100)
[<c028feb8>] (driver_detach) from [<c028f1f4>] (bus_remove_driver+0x4c/0xa0)
[<c028f1f4>] (bus_remove_driver) from [<bf2fcdd8>] (rt_mxs_auart_exit+0xc/0x20 [moduleXenoUartSimple])
[<bf2fcdd8>] (rt_mxs_auart_exit [moduleXenoUartSimple]) from [<c006fd5c>] (SyS_delete_module+0x15c/0x1cc)
[<c006fd5c>] (SyS_delete_module) from [<c000a500>] (ret_fast_syscall+0x0/0x38)
---[ end trace 94d65edb72d5735d ]---

This doesn't always happen and sometimes I can rmmod it without any problem. I suspect that this is related to the clock because of the message WARNING: CPU: 0 PID: 825 at drivers/clk/clk.c:130 clk_enable_lock

If useful, here is part of the driver code that I'm using (inspired by mxs-auart.c and rt_imx_uart.c)

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("IMX28 UART RTDM driver");

struct rt_mxs_auart_port {

    unsigned long flags;
    unsigned int mctrl_prev;
    enum mxs_auart_type devtype;
    const struct vendor_data *vendor;

    struct clk *clk;
    struct clk *clk_ahb;


    unsigned char __iomem *membase; /* read/write[bwl] */
    resource_size_t     mapbase;        /* for ioremap */
    unsigned int irq;       /* irq number */
    int fifosize;            /* TX fifo size*/
    unsigned int uartclk;       /* base uart clock */
    struct rtdm_device rtdm_dev;    /* RTDM device structure */
};

static int mxs_get_clks(struct rt_mxs_auart_port *s, struct platform_device *pdev)
{
    int err;

    if (!is_asm9260_auart(s)) {
        s->clk = devm_clk_get(&pdev->dev, NULL);
        return PTR_ERR_OR_ZERO(s->clk);
    }

    s->clk = devm_clk_get(&pdev->dev, "mod");
    if (IS_ERR(s->clk)) {
        pr_err("Failed to get \"mod\" clk\n");
        return PTR_ERR(s->clk);
    }

    s->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
    if (IS_ERR(s->clk_ahb)) {
        pr_err("Failed to get \"ahb\" clk\n");
        return PTR_ERR(s->clk_ahb);
    }

    err = clk_prepare_enable(s->clk_ahb);
    if (err) {
        pr_err("Failed to enable ahb_clk!\n");
        return err;
    }

    /*
    err = clk_set_rate(s->clk, clk_get_rate(s->clk_ahb));
    if (err) {
        dev_err(s->dev, "Failed to set rate!\n");
        goto disable_clk_ahb;
    }
    */
    err = clk_prepare_enable(s->clk);
    if (err) {
        pr_err("Failed to enable clk!\n");
        goto disable_clk_ahb;
    }


    return 0;

disable_clk_ahb:
    clk_disable_unprepare(s->clk_ahb);
    return err;
}

static int rt_serial_mxs_probe_dt(struct rt_mxs_auart_port *s,
        struct platform_device *pdev)
{
    struct device_node *np = pdev->dev.of_node;
    int ret;

    if (!np)
        /* no device tree device */
        return 1;

    ret = of_alias_get_id(np, "serial");
    if (ret < 0) {
        dev_err(&pdev->dev, "failed to get alias id: %d\n", ret);
        return ret;
    }
    pdev->id = ret;

    //s->port.line = ret;


    return 0;
}

static int rt_mxs_auart_probe(struct platform_device *pdev)
{

    const struct of_device_id *of_id =
            of_match_device(mxs_auart_dt_ids, &pdev->dev);
    //u32 version;
    int ret;
    //int irq;
    struct resource *r;

    //we're adding this
    struct rtdm_device *dev;
    struct rt_mxs_auart_port *s;

    //allocate managed kernel memory
    s = devm_kzalloc(&pdev->dev, sizeof(*s), GFP_KERNEL);
    if (!s)
        return -ENOMEM;

    //s->port.dev = &pdev->dev;
    //s->dev = &pdev->dev;

    ret = rt_serial_mxs_probe_dt(s, pdev);
    if (ret > 0)
        pr_info("If ret>0\n");
        //s->port.line = pdev->id < 0 ? 0 : pdev->id;//what do we do here?
    else if (ret < 0){
        return ret;
        }

    if (of_id) {
        pdev->id_entry = of_id->data;
        //s->devdata = of_id->data;
        s->devtype = pdev->id_entry->driver_data;
    }

    r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
    if (!r)
        return -ENXIO;

    //get irq line
    s->irq = platform_get_irq(pdev, 0);
    if (s->irq < 0)
        return -ENODEV;

    //base memory
    //s->membase = devm_ioremap_resource(&pdev->dev, r);
    s->membase = ioremap(r->start, resource_size(r));
    if (IS_ERR(s->membase))
        return PTR_ERR(s->membase);

    /*define the rtdm_device*/
    dev = &s->rtdm_dev;
    dev->driver = &ida_driver;
    dev->label = "xeno_sprinte_ida";
    dev->device_data = s;

    ret = mxs_get_clks(s, pdev); //activate clocks
    if (ret)
        return ret;
    s->uartclk = clk_get_rate(s->clk);

    s->fifosize = MXS_AUART_FIFO_SIZE;

    mxs_init_regs(s);

    ret = rtdm_dev_register(dev); //register driver in RTDM space
    if (ret)
        return ret;

    platform_set_drvdata(pdev, s); //add device-related data to device struct

    pr_info("Probe: Successful\n");
    pr_info("%s on AUART%d: physical address=0x%p membase=0x%p irq=%d uartclk=%d\n",
           dev->name, pdev->id, r, s->membase, s->irq, s->uartclk);
    return 0;


}

static int rt_mxs_auart_remove(struct platform_device *pdev)
{
    struct rt_mxs_auart_port *s = platform_get_drvdata(pdev);
    struct rtdm_device *dev = &s->rtdm_dev;

    platform_set_drvdata(pdev, NULL);

    clk_disable_unprepare(s->clk);
    clk_disable_unprepare(s->clk_ahb);
    rtdm_dev_unregister(dev);

    pr_info("Remove: Successful\n");
    return 0;
}


static struct platform_driver rt_mxs_auart_driver = {
    .probe = rt_mxs_auart_probe,
    .remove = rt_mxs_auart_remove,
    .driver = {
        .name = "xeno_sprinte_ida",
        .of_match_table = mxs_auart_dt_ids,
    },
};


static int __init rt_mxs_auart_init(void)
{
    int ret;

    /*what library to include?
    if (!rtdm_available())
    return -ENODEV;
    */

    ret = platform_driver_register(&rt_mxs_auart_driver);
    if (ret) {
        pr_err("%s; Could not register  driver (err=%d)\n",
            __func__, ret);
    }

    pr_info("Init: Successful\n");
    return ret;
}

static void __exit rt_mxs_auart_exit(void)
{
    platform_driver_unregister(&rt_mxs_auart_driver);
    pr_info("Exit: Successful\n");
}

module_init(rt_mxs_auart_init);
module_exit(rt_mxs_auart_exit);

Does anyone have an idea? Thanks!