/*******************************************************************************
    Copyright (c) 2016-2021 NVIDIA Corporation

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
    deal in the Software without restriction, including without limitation the
    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
    sell copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

        The above copyright notice and this permission notice shall be
        included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.

*******************************************************************************/
#include "linux_nvswitch.h"

#include <linux/version.h>

#include "conftest.h"
#include "nvlink_errors.h"
#include "nvlink_linux.h"
#include "nvCpuUuid.h"
#include "nv-time.h"
#include "nvlink_caps.h"

#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/cdev.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/poll.h>
#include <linux/sched.h>
#include <linux/time.h>
#include <linux/string.h>
#include <linux/moduleparam.h>
#include <linux/ctype.h>
#include <linux/wait.h>
#include <linux/jiffies.h>

#include "ioctl_nvswitch.h"

const static struct
{
    NvlStatus status;
    int err;
} nvswitch_status_map[] = {
    { NVL_ERR_GENERIC,                  -EIO        },
    { NVL_NO_MEM,                       -ENOMEM     },
    { NVL_BAD_ARGS,                     -EINVAL     },
    { NVL_ERR_INVALID_STATE,            -EIO        },
    { NVL_ERR_NOT_SUPPORTED,            -EOPNOTSUPP },
    { NVL_NOT_FOUND,                    -EINVAL     },
    { NVL_ERR_STATE_IN_USE,             -EBUSY      },
    { NVL_ERR_NOT_IMPLEMENTED,          -ENOSYS     },
    { NVL_ERR_INSUFFICIENT_PERMISSIONS, -EPERM      },
    { NVL_ERR_OPERATING_SYSTEM,         -EIO        },
    { NVL_MORE_PROCESSING_REQUIRED,     -EAGAIN     },
    { NVL_SUCCESS,                       0          },
};

int
nvswitch_map_status
(
    NvlStatus status
)
{
    int err = -EIO;
    NvU32 i;
    NvU32 limit = sizeof(nvswitch_status_map) / sizeof(nvswitch_status_map[0]);

    for (i = 0; i < limit; i++)
    {
        if (nvswitch_status_map[i].status == status ||
            nvswitch_status_map[i].status == -status)
        {
            err = nvswitch_status_map[i].err;
            break;
        }
    }

    return err;
}

#if !defined(IRQF_SHARED)
#define IRQF_SHARED SA_SHIRQ
#endif

#if defined(NV_FILE_HAS_INODE)
#define NV_FILE_INODE(file) (file)->f_inode
#else
#define NV_FILE_INODE(file) (file)->f_dentry->d_inode
#endif

static int nvswitch_probe(struct pci_dev *, const struct pci_device_id *);
static void nvswitch_remove(struct pci_dev *);

static struct pci_device_id nvswitch_pci_table[] =
{
    {
        .vendor      = PCI_VENDOR_ID_NVIDIA,
        .device      = PCI_ANY_ID,
        .subvendor   = PCI_ANY_ID,
        .subdevice   = PCI_ANY_ID,
        .class       = (PCI_CLASS_BRIDGE_OTHER << 8),
        .class_mask  = ~0
    },
    {}
};

static struct pci_driver nvswitch_pci_driver =
{
    .name           = NVSWITCH_DRIVER_NAME,
    .id_table       = nvswitch_pci_table,
    .probe          = nvswitch_probe,
    .remove         = nvswitch_remove,
    .shutdown       = nvswitch_remove
};

//
// nvidia_nvswitch_mknod uses minor number 255 to create nvidia-nvswitchctl
// node. Hence, if NVSWITCH_CTL_MINOR is changed, then NV_NVSWITCH_CTL_MINOR
// should be updated. See nvdia-modprobe-utils.h
//
#define NVSWITCH_CTL_MINOR 255
#define NVSWITCH_MINOR_COUNT (NVSWITCH_CTL_MINOR + 1)

// 32 bit hex value - including 0x prefix. (10 chars)
#define NVSWITCH_REGKEY_VALUE_LEN 10

static char *NvSwitchRegDwords;
module_param(NvSwitchRegDwords, charp, 0);
MODULE_PARM_DESC(NvSwitchRegDwords, "NvSwitch regkey");

static char *NvSwitchBlacklist;
module_param(NvSwitchBlacklist, charp, 0);
MODULE_PARM_DESC(NvSwitchBlacklist, "NvSwitchBlacklist=uuid[,uuid...]");

//
// Locking:
//   We handle nvswitch driver locking in the OS layer. The nvswitch lib
//   layer does not have its own locking. It relies on the OS layer for
//   atomicity.
//
//   All locking is done with sleep locks. We use threaded MSI interrupts to
//   facilitate this.
//
//   When handling a request from a user context we use the interruptible
//   version to enable a quick ^C return if there is lock contention.
//
//   nvswitch.driver_mutex is used to protect driver's global state, "struct
//   NVSWITCH". The driver_mutex is taken during .probe, .remove, .open,
//   .close, and nvswitch-ctl .ioctl operations.
//
//   nvswitch_dev.device_mutex is used to protect per-device state, "struct
//   NVSWITCH_DEV", once a device is opened. The device_mutex is taken during
//   .ioctl, .poll and other background tasks.
//
//   The kernel guarantees that .close won't happen while .ioctl and .poll
//   are going on and without successful .open one can't execute any file ops.
//   This behavior guarantees correctness of the locking model.
//
//   If .close is invoked and holding the lock which is also used by threaded
//   tasks such as interrupt, driver will deadlock while trying to stop such
//   tasks. For example, when threaded interrupts are enabled, free_irq() calls
//   kthread_stop() to flush pending interrupt tasks. The locking model
//   makes sure that such deadlock cases don't happen.
//
// Lock ordering:
//   nvswitch.driver_mutex
//   nvswitch_dev.device_mutex
//
// Note:
//   Due to bug 2856314, nvswitch_dev.device_mutex is taken when calling
//   nvswitch_post_init_device() in nvswitch_probe().
//

// Per-chip driver state is defined in linux_nvswitch.h

// Global driver state
typedef struct
{
    NvBool initialized;
    struct cdev cdev;
    struct cdev cdev_ctl;
    dev_t devno;
    atomic_t count;
    struct mutex driver_mutex;
    struct list_head devices;
} NVSWITCH;

static NVSWITCH nvswitch = {0};

// NvSwitch event
typedef struct nvswitch_event_t
{
    wait_queue_head_t wait_q_event;
    NvBool            event_pending;
} nvswitch_event_t;

typedef struct nvswitch_file_private
{
    NVSWITCH_DEV     *nvswitch_dev;
    nvswitch_event_t file_event;
    struct
    {
        /* A duped file descriptor for fabric_mgmt capability */
        int fabric_mgmt;
    } capability_fds;
} nvswitch_file_private_t;

#define NVSWITCH_SET_FILE_PRIVATE(filp, data) ((filp)->private_data = (data))
#define NVSWITCH_GET_FILE_PRIVATE(filp) ((nvswitch_file_private_t *)(filp)->private_data)

static int nvswitch_device_open(struct inode *inode, struct file *file);
static int nvswitch_device_release(struct inode *inode, struct file *file);
static unsigned int nvswitch_device_poll(struct file *file, poll_table *wait);
static int nvswitch_device_ioctl(struct inode *inode,
                                 struct file *file,
                                 unsigned int cmd,
                                 unsigned long arg);
static long nvswitch_device_unlocked_ioctl(struct file *file,
                                           unsigned int cmd,
                                           unsigned long arg);

static int nvswitch_ctl_ioctl(struct inode *inode,
                              struct file *file,
                              unsigned int cmd,
                              unsigned long arg);
static long nvswitch_ctl_unlocked_ioctl(struct file *file,
                                        unsigned int cmd,
                                        unsigned long arg);

struct file_operations device_fops =
{
    .owner = THIS_MODULE,
#if defined(NV_FILE_OPERATIONS_HAS_IOCTL)
    .ioctl = nvswitch_device_ioctl,
#endif
    .unlocked_ioctl = nvswitch_device_unlocked_ioctl,
    .open    = nvswitch_device_open,
    .release = nvswitch_device_release,
    .poll    = nvswitch_device_poll
};

struct file_operations ctl_fops =
{
    .owner = THIS_MODULE,
#if defined(NV_FILE_OPERATIONS_HAS_IOCTL)
    .ioctl = nvswitch_ctl_ioctl,
#endif
    .unlocked_ioctl = nvswitch_ctl_unlocked_ioctl,
};

static int nvswitch_initialize_device_interrupt(NVSWITCH_DEV *nvswitch_dev);
static void nvswitch_shutdown_device_interrupt(NVSWITCH_DEV *nvswitch_dev);
static void nvswitch_load_bar_info(NVSWITCH_DEV *nvswitch_dev);
static void nvswitch_task_dispatch(NVSWITCH_DEV *nvswitch_dev);

static NvBool
nvswitch_is_device_blacklisted
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    NVSWITCH_DEVICE_FABRIC_STATE device_fabric_state = 0;
    NvlStatus status;

    status = nvswitch_lib_read_fabric_state(nvswitch_dev->lib_device, 
                                            &device_fabric_state, NULL, NULL);

    if (status != NVL_SUCCESS)
    {
        printk(KERN_INFO "%s: Failed to read fabric state, %x\n", nvswitch_dev->name, status);
        return NV_FALSE;
    }

    return device_fabric_state == NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED;
}

static void
nvswitch_deinit_background_tasks
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    NV_ATOMIC_SET(nvswitch_dev->task_q_ready, 0);

    wake_up(&nvswitch_dev->wait_q_shutdown);

    nv_kthread_q_stop(&nvswitch_dev->task_q);
}

static int
nvswitch_init_background_tasks
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    int rc;

    rc = nv_kthread_q_init(&nvswitch_dev->task_q, nvswitch_dev->sname);
    if (rc)
    {
        printk(KERN_ERR "%s: Failed to create task queue\n", nvswitch_dev->name);
        return rc;
    }

    NV_ATOMIC_SET(nvswitch_dev->task_q_ready, 1);

    nv_kthread_q_item_init(&nvswitch_dev->task_item,
                           (nv_q_func_t) &nvswitch_task_dispatch,
                           nvswitch_dev);

    if (!nv_kthread_q_schedule_q_item(&nvswitch_dev->task_q,
                                      &nvswitch_dev->task_item))
    {
        printk(KERN_ERR "%s: Failed to schedule an item\n",nvswitch_dev->name);
        rc = -ENODEV;
        goto init_background_task_failed;
    }

    return 0;

init_background_task_failed:
    nvswitch_deinit_background_tasks(nvswitch_dev);

    return rc;
}

static NVSWITCH_DEV*
nvswitch_find_device(int minor)
{
    struct list_head *cur;
    NVSWITCH_DEV *nvswitch_dev = NULL;

    list_for_each(cur, &nvswitch.devices)
    {
        nvswitch_dev = list_entry(cur, NVSWITCH_DEV, list_node);
        if (nvswitch_dev->minor == minor)
        {
            return nvswitch_dev;
        }
    }

    return NULL;
}

static int
nvswitch_find_minor(void)
{
    struct list_head *cur;
    NVSWITCH_DEV *nvswitch_dev;
    int minor;
    int minor_in_use;

    for (minor = 0; minor < NVSWITCH_DEVICE_INSTANCE_MAX; minor++)
    {
        minor_in_use = 0;

        list_for_each(cur, &nvswitch.devices)
        {
            nvswitch_dev = list_entry(cur, NVSWITCH_DEV, list_node);
            if (nvswitch_dev->minor == minor)
            {
                minor_in_use = 1;
                break;
            }
        }

        if (!minor_in_use)
        {
            return minor;
        }
    }

    return NVSWITCH_DEVICE_INSTANCE_MAX;
}

static int
nvswitch_init_i2c_adapters
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    NvlStatus retval;
    NvU32 i, valid_ports_mask;
    struct i2c_adapter *adapter;
    nvswitch_i2c_adapter_entry *adapter_entry;

    if (!nvswitch_lib_is_i2c_supported(nvswitch_dev->lib_device))
    {
        return 0;
    }

    retval = nvswitch_lib_get_valid_ports_mask(nvswitch_dev->lib_device,
                                               &valid_ports_mask);
    if (retval != NVL_SUCCESS)
    {
        printk(KERN_ERR "Failed to get valid I2C ports mask.\n");
        return -ENODEV;
    }

    FOR_EACH_INDEX_IN_MASK(32, i, valid_ports_mask)
    {
        adapter = nvswitch_i2c_add_adapter(nvswitch_dev, i);
        if (adapter == NULL)
        {
            continue;
        }

        adapter_entry = nvswitch_os_malloc(sizeof(*adapter_entry));
        if (adapter_entry == NULL)
        {
            printk(KERN_ERR "Failed to create I2C adapter entry.\n");
            nvswitch_i2c_del_adapter(adapter);
            continue;
        }

        adapter_entry->adapter = adapter;

        list_add_tail(&adapter_entry->entry, &nvswitch_dev->i2c_adapter_list);
    }
    FOR_EACH_INDEX_IN_MASK_END;

    return 0;
}

static void
nvswitch_deinit_i2c_adapters
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    nvswitch_i2c_adapter_entry *curr;
    nvswitch_i2c_adapter_entry *next;

    list_for_each_entry_safe(curr,
                             next,
                             &nvswitch_dev->i2c_adapter_list,
                             entry)
    {
        nvswitch_i2c_del_adapter(curr->adapter);
        list_del(&curr->entry);
        nvswitch_os_free(curr);
    }
}

static int
nvswitch_init_device
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    struct pci_dev *pci_dev = nvswitch_dev->pci_dev;
    NvlStatus retval;
    int rc;

    INIT_LIST_HEAD(&nvswitch_dev->i2c_adapter_list);

    retval = nvswitch_lib_register_device(NV_PCI_DOMAIN_NUMBER(pci_dev),
                                          NV_PCI_BUS_NUMBER(pci_dev),
                                          NV_PCI_SLOT_NUMBER(pci_dev),
                                          PCI_FUNC(pci_dev->devfn),
                                          pci_dev->device,
                                          pci_dev,
                                          nvswitch_dev->minor,
                                          &nvswitch_dev->lib_device);
    if (NVL_SUCCESS != retval)
    {
        printk(KERN_ERR "%s: Failed to register device : %d\n",
               nvswitch_dev->name,
               retval);
        return -ENODEV;
    }

    nvswitch_load_bar_info(nvswitch_dev);

    retval = nvswitch_lib_initialize_device(nvswitch_dev->lib_device);
    if (NVL_SUCCESS != retval)
    {
        printk(KERN_ERR "%s: Failed to initialize device : %d\n",
               nvswitch_dev->name,
               retval);
        rc = -ENODEV;
        goto init_device_failed;
    }

    nvswitch_lib_get_uuid(nvswitch_dev->lib_device, &nvswitch_dev->uuid);

    if (nvswitch_lib_get_bios_version(nvswitch_dev->lib_device,
                                      &nvswitch_dev->bios_ver) != NVL_SUCCESS)
    {
        nvswitch_dev->bios_ver = 0;
    }

    if (nvswitch_lib_get_physid(nvswitch_dev->lib_device,
                                &nvswitch_dev->phys_id) != NVL_SUCCESS)
    {
        nvswitch_dev->phys_id = NVSWITCH_INVALID_PHYS_ID;
    }

    rc = nvswitch_initialize_device_interrupt(nvswitch_dev);
    if (rc)
    {
        printk(KERN_ERR "%s: Failed to initialize interrupt : %d\n",
               nvswitch_dev->name,
               rc);
        goto init_intr_failed;
    }

    if (nvswitch_is_device_blacklisted(nvswitch_dev))
    {
        printk(KERN_ERR "%s: Blacklisted nvswitch device\n", nvswitch_dev->name);
        // Keep device registered for HAL access and Fabric State updates
        return 0;
    }

    nvswitch_lib_enable_interrupts(nvswitch_dev->lib_device);

    return 0;

init_intr_failed:
    nvswitch_lib_shutdown_device(nvswitch_dev->lib_device);

init_device_failed:
    nvswitch_lib_unregister_device(nvswitch_dev->lib_device);
    nvswitch_dev->lib_device = NULL;

    return rc;
}

static int
nvswitch_post_init_device
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    int rc;
    NvlStatus retval;

    rc = nvswitch_init_i2c_adapters(nvswitch_dev);
    if (rc < 0)
    {
       return rc;
    }

    retval = nvswitch_lib_post_init_device(nvswitch_dev->lib_device);
    if (retval != NVL_SUCCESS)
    {
        return -ENODEV;
    }

    return 0;
}

static void
nvswitch_post_init_blacklisted
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    nvswitch_lib_post_init_blacklist_device(nvswitch_dev->lib_device);
}

static void
nvswitch_deinit_device
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    nvswitch_lib_disable_interrupts(nvswitch_dev->lib_device);

    nvswitch_shutdown_device_interrupt(nvswitch_dev);

    nvswitch_lib_shutdown_device(nvswitch_dev->lib_device);

    nvswitch_lib_unregister_device(nvswitch_dev->lib_device);
    nvswitch_dev->lib_device = NULL;
}

static void
nvswitch_init_file_event
(
    nvswitch_file_private_t *private
)
{
    init_waitqueue_head(&private->file_event.wait_q_event);
    private->file_event.event_pending = NV_FALSE;
}

//
// Basic device open to support IOCTL interface
//
static int
nvswitch_device_open
(
    struct inode *inode,
    struct file *file
)
{
    NVSWITCH_DEV *nvswitch_dev;
    int rc = 0;
    nvswitch_file_private_t *private = NULL;

    //
    // Get the major/minor device
    // We might want this for routing requests to multiple nvswitches
    //
    printk(KERN_INFO "nvidia-nvswitch%d: open (major=%d)\n",
           MINOR(inode->i_rdev),
           MAJOR(inode->i_rdev));

    rc = mutex_lock_interruptible(&nvswitch.driver_mutex);
    if (rc)
    {
        return rc;
    }

    nvswitch_dev = nvswitch_find_device(MINOR(inode->i_rdev));
    if (!nvswitch_dev)
    {
        rc = -ENODEV;
        goto done;
    }

    if (nvswitch_is_device_blacklisted(nvswitch_dev))
    {
        rc = -ENODEV;
        goto done;
    }

    private = nvswitch_os_malloc(sizeof(*private));
    if (private == NULL)
    {
        rc = -ENOMEM;
        goto done;
    }

    private->nvswitch_dev = nvswitch_dev;

    nvswitch_init_file_event(private);

    private->capability_fds.fabric_mgmt = -1;
    NVSWITCH_SET_FILE_PRIVATE(file, private);

    NV_ATOMIC_INC(nvswitch_dev->ref_count);

done:
    mutex_unlock(&nvswitch.driver_mutex);

    return rc;
}

//
// Basic device release to support IOCTL interface
//
static int
nvswitch_device_release
(
    struct inode *inode,
    struct file *file
)
{
    nvswitch_file_private_t *private = NVSWITCH_GET_FILE_PRIVATE(file);
    NVSWITCH_DEV *nvswitch_dev = private->nvswitch_dev;

    printk(KERN_INFO "nvidia-nvswitch%d: release (major=%d)\n",
           MINOR(inode->i_rdev),
           MAJOR(inode->i_rdev));

    mutex_lock(&nvswitch.driver_mutex);

    nvswitch_lib_remove_client_events(nvswitch_dev->lib_device, (void *)private);

    //
    // If there are no outstanding references and the device is marked
    // unusable, free it.
    //
    if (NV_ATOMIC_DEC_AND_TEST(nvswitch_dev->ref_count) &&
        nvswitch_dev->unusable)
    {
        kfree(nvswitch_dev);
    }

    if (private->capability_fds.fabric_mgmt > 0)
    {
        nvlink_cap_release(private->capability_fds.fabric_mgmt);
        private->capability_fds.fabric_mgmt = -1;
    }

    nvswitch_os_free(file->private_data);
    NVSWITCH_SET_FILE_PRIVATE(file, NULL);

    mutex_unlock(&nvswitch.driver_mutex);

    return 0;
}

static unsigned int
nvswitch_device_poll
(
    struct file *file,
    poll_table *wait
)
{
    nvswitch_file_private_t *private = NVSWITCH_GET_FILE_PRIVATE(file);
    NVSWITCH_DEV *nvswitch_dev = private->nvswitch_dev;
    int rc = 0;
    NvlStatus status;
    struct NVSWITCH_CLIENT_EVENT *client_event;

    rc = mutex_lock_interruptible(&nvswitch_dev->device_mutex);
    if (rc)
    {
        return rc;
    }

    if (nvswitch_dev->unusable)
    {
        printk(KERN_INFO "%s: a stale fd detected\n", nvswitch_dev->name);
        rc = POLLHUP;
        goto done;
    }

    status = nvswitch_lib_get_client_event(nvswitch_dev->lib_device,
                                           (void *) private, &client_event);
    if (status != NVL_SUCCESS)
    {
        printk(KERN_INFO "%s: no events registered for fd\n", nvswitch_dev->name);
        rc = POLLERR;
        goto done;
    }

    poll_wait(file, &private->file_event.wait_q_event, wait);

    if (private->file_event.event_pending)
    {
        rc = POLLPRI | POLLIN;
        private->file_event.event_pending = NV_FALSE;
    }

done:
    mutex_unlock(&nvswitch_dev->device_mutex);

    return rc;
}

typedef struct {
    void *kernel_params;                // Kernel copy of ioctl parameters
    unsigned long kernel_params_size;   // Size of ioctl params according to user
} IOCTL_STATE;

//
// Clean up any dynamically allocated memory for ioctl state
//
static void
nvswitch_ioctl_state_cleanup
(
    IOCTL_STATE *state
)
{
    kfree(state->kernel_params);
    state->kernel_params = NULL;
}

//
// Initialize buffer state for ioctl.
//
// This handles allocating memory and copying user data into kernel space.  The
// ioctl params structure only is supported. Nested data pointers are not handled.
//
// State is maintained in the IOCTL_STATE struct for use by the ioctl, _sync and
// _cleanup calls.
//
static int
nvswitch_ioctl_state_start(IOCTL_STATE *state, int cmd, unsigned long user_arg)
{
    int rc;

    state->kernel_params = NULL;
    state->kernel_params_size = _IOC_SIZE(cmd);

    if (0 == state->kernel_params_size)
    {
        return 0;
    }

    state->kernel_params = kzalloc(state->kernel_params_size, GFP_KERNEL);
    if (NULL == state->kernel_params)
    {
        rc = -ENOMEM;
        goto nvswitch_ioctl_state_start_fail;
    }

    // Copy params to kernel buffers.  Simple _IOR() ioctls can skip this step.
    if (_IOC_DIR(cmd) & _IOC_WRITE)
    {
        rc = copy_from_user(state->kernel_params,
                            (const void *)user_arg,
                            state->kernel_params_size);
        if (rc)
        {
            rc = -EFAULT;
            goto nvswitch_ioctl_state_start_fail;
        }
    }

    return 0;

nvswitch_ioctl_state_start_fail:
    nvswitch_ioctl_state_cleanup(state);
    return rc;
}

//
// Synchronize any ioctl output in the kernel buffers to the user mode buffers.
//
static int
nvswitch_ioctl_state_sync
(
    IOCTL_STATE *state,
    int cmd,
    unsigned long user_arg
)
{
    int rc;

    // Nothing to do if no buffer or write-only ioctl
    if ((0 == state->kernel_params_size) || (0 == (_IOC_DIR(cmd) & _IOC_READ)))
    {
        return 0;
    }

    // Copy params structure back to user mode
    rc = copy_to_user((void *)user_arg,
                      state->kernel_params,
                      state->kernel_params_size);
    if (rc)
    {
        rc = -EFAULT;
    }

    return rc;
}

static int
nvswitch_device_ioctl
(
    struct inode *inode,
    struct file *file,
    unsigned int cmd,
    unsigned long arg
)
{
    nvswitch_file_private_t *private = NVSWITCH_GET_FILE_PRIVATE(file);
    NVSWITCH_DEV *nvswitch_dev = private->nvswitch_dev;
    IOCTL_STATE state = {0};
    NvlStatus retval;
    int rc = 0;

    if (_IOC_TYPE(cmd) != NVSWITCH_DEV_IO_TYPE)
    {
        return -EINVAL;
    }

    rc = mutex_lock_interruptible(&nvswitch_dev->device_mutex);
    if (rc)
    {
        return rc;
    }

    if (nvswitch_dev->unusable)
    {
        printk(KERN_INFO "%s: a stale fd detected\n", nvswitch_dev->name);
        rc = -ENODEV;
        goto nvswitch_device_ioctl_exit;
    }

    if (nvswitch_is_device_blacklisted(nvswitch_dev))
    {
        printk(KERN_INFO "%s: ioctl attempted on blacklisted device\n", nvswitch_dev->name);
        rc = -ENODEV;
        goto nvswitch_device_ioctl_exit;
    }

    rc = nvswitch_ioctl_state_start(&state, cmd, arg);
    if (rc)
    {
        goto nvswitch_device_ioctl_exit;
    }

    retval = nvswitch_lib_ctrl(nvswitch_dev->lib_device,
                               _IOC_NR(cmd),
                               state.kernel_params,
                               state.kernel_params_size,
                               file->private_data);
    rc = nvswitch_map_status(retval);
    if (!rc)
    {
        rc = nvswitch_ioctl_state_sync(&state, cmd, arg);
    }

    nvswitch_ioctl_state_cleanup(&state);

nvswitch_device_ioctl_exit:
    mutex_unlock(&nvswitch_dev->device_mutex);

    return rc;
}

static long
nvswitch_device_unlocked_ioctl
(
    struct file *file,
    unsigned int cmd,
    unsigned long arg
)
{
    return nvswitch_device_ioctl(NV_FILE_INODE(file), file, cmd, arg);
}

static int
nvswitch_ctl_check_version(NVSWITCH_CHECK_VERSION_PARAMS *p)
{
    NvlStatus retval;

    p->is_compatible = 0;
    p->user.version[NVSWITCH_VERSION_STRING_LENGTH - 1] = '\0';

    retval = nvswitch_lib_check_api_version(p->user.version, p->kernel.version,
                                            NVSWITCH_VERSION_STRING_LENGTH);
    if (retval == NVL_SUCCESS)
    {
        p->is_compatible = 1;
    }
    else if (retval == -NVL_ERR_NOT_SUPPORTED)
    {
        printk(KERN_ERR "nvidia-nvswitch: Version mismatch, "
               "kernel version %s user version %s\n",
               p->kernel.version, p->user.version);
    }
    else
    {
        // An unexpected failure
        return nvswitch_map_status(retval);
    }

    return 0;
}

static void
nvswitch_ctl_get_devices(NVSWITCH_GET_DEVICES_PARAMS *p)
{
    int index = 0;
    NVSWITCH_DEV *nvswitch_dev;
    struct list_head *cur;

    BUILD_BUG_ON(NVSWITCH_DEVICE_INSTANCE_MAX != NVSWITCH_MAX_DEVICES);

    list_for_each(cur, &nvswitch.devices)
    {
        nvswitch_dev = list_entry(cur, NVSWITCH_DEV, list_node);
        p->info[index].deviceInstance = nvswitch_dev->minor;
        p->info[index].pciDomain = NV_PCI_DOMAIN_NUMBER(nvswitch_dev->pci_dev);
        p->info[index].pciBus = NV_PCI_BUS_NUMBER(nvswitch_dev->pci_dev);
        p->info[index].pciDevice = NV_PCI_SLOT_NUMBER(nvswitch_dev->pci_dev);
        p->info[index].pciFunction = PCI_FUNC(nvswitch_dev->pci_dev->devfn);
        index++;
    }

    p->deviceCount = index;
}

static void
nvswitch_ctl_get_devices_v2(NVSWITCH_GET_DEVICES_V2_PARAMS *p)
{
    int index = 0;
    NVSWITCH_DEV *nvswitch_dev;
    struct list_head *cur;

    BUILD_BUG_ON(NVSWITCH_DEVICE_INSTANCE_MAX != NVSWITCH_MAX_DEVICES);

    list_for_each(cur, &nvswitch.devices)
    {
        nvswitch_dev = list_entry(cur, NVSWITCH_DEV, list_node);
        p->info[index].deviceInstance = nvswitch_dev->minor;
        memcpy(&p->info[index].uuid, &nvswitch_dev->uuid, sizeof(nvswitch_dev->uuid));
        p->info[index].pciDomain = NV_PCI_DOMAIN_NUMBER(nvswitch_dev->pci_dev);
        p->info[index].pciBus = NV_PCI_BUS_NUMBER(nvswitch_dev->pci_dev);
        p->info[index].pciDevice = NV_PCI_SLOT_NUMBER(nvswitch_dev->pci_dev);
        p->info[index].pciFunction = PCI_FUNC(nvswitch_dev->pci_dev->devfn);
        p->info[index].physId = nvswitch_dev->phys_id;

        if (nvswitch_dev->lib_device != NULL)
        {
            mutex_lock(&nvswitch_dev->device_mutex);
            (void)nvswitch_lib_read_fabric_state(nvswitch_dev->lib_device,
                                                 &p->info[index].deviceState,
                                                 &p->info[index].deviceReason,
                                                 &p->info[index].driverState);
            mutex_unlock(&nvswitch_dev->device_mutex);
        }
        index++;
    }

    p->deviceCount = index;
}

#define NVSWITCH_CTL_CHECK_PARAMS(type, size) (sizeof(type) == size ? 0 : -EINVAL)

static int
nvswitch_ctl_cmd_dispatch
(
    unsigned int cmd,
    void *params,
    unsigned int param_size
)
{
    int rc;

    switch(cmd)
    {
        case CTRL_NVSWITCH_CHECK_VERSION:
            rc = NVSWITCH_CTL_CHECK_PARAMS(NVSWITCH_CHECK_VERSION_PARAMS,
                                           param_size);
            if (!rc)
            {
                rc = nvswitch_ctl_check_version(params);
            }
            break;
        case CTRL_NVSWITCH_GET_DEVICES:
            rc = NVSWITCH_CTL_CHECK_PARAMS(NVSWITCH_GET_DEVICES_PARAMS,
                                           param_size);
            if (!rc)
            {
                nvswitch_ctl_get_devices(params);
            }
            break;
        case CTRL_NVSWITCH_GET_DEVICES_V2:
            rc = NVSWITCH_CTL_CHECK_PARAMS(NVSWITCH_GET_DEVICES_V2_PARAMS,
                                           param_size);
            if (!rc)
            {
                nvswitch_ctl_get_devices_v2(params);
            }
            break;

        default:
            rc = -EINVAL;
            break;
    }

    return rc;
}

static int
nvswitch_ctl_ioctl
(
    struct inode *inode,
    struct file *file,
    unsigned int cmd,
    unsigned long arg
)
{
    int rc = 0;
    IOCTL_STATE state = {0};

    if (_IOC_TYPE(cmd) != NVSWITCH_CTL_IO_TYPE)
    {
        return -EINVAL;
    }

    rc = mutex_lock_interruptible(&nvswitch.driver_mutex);
    if (rc)
    {
        return rc;
    }

    rc = nvswitch_ioctl_state_start(&state, cmd, arg);
    if (rc)
    {
        goto nvswitch_ctl_ioctl_exit;
    }

    rc = nvswitch_ctl_cmd_dispatch(_IOC_NR(cmd),
                                   state.kernel_params,
                                   state.kernel_params_size);
    if (!rc)
    {
        rc = nvswitch_ioctl_state_sync(&state, cmd, arg);
    }

    nvswitch_ioctl_state_cleanup(&state);

nvswitch_ctl_ioctl_exit:
    mutex_unlock(&nvswitch.driver_mutex);

    return rc;
}

static long
nvswitch_ctl_unlocked_ioctl
(
    struct file *file,
    unsigned int cmd,
    unsigned long arg
)
{
    return nvswitch_ctl_ioctl(NV_FILE_INODE(file), file, cmd, arg);
}

static irqreturn_t
nvswitch_isr_pending
(
    int   irq,
    void *arg
)
{

    NVSWITCH_DEV *nvswitch_dev = (NVSWITCH_DEV *)arg;
    NvlStatus retval;

    //
    // On silicon MSI must be enabled.  Since interrupts will not be shared
    // with MSI, we can simply signal the thread.
    //
    if (nvswitch_dev->irq_mechanism == NVSWITCH_IRQ_MSI)
    {
        return IRQ_WAKE_THREAD;
    }

    if (nvswitch_dev->irq_mechanism == NVSWITCH_IRQ_PIN)
    {
        //
        // We do not take mutex in the interrupt context. The interrupt
        // check is safe to driver state.
        //
        retval = nvswitch_lib_check_interrupts(nvswitch_dev->lib_device);

        // Wake interrupt thread if there is an interrupt pending
        if (-NVL_MORE_PROCESSING_REQUIRED == retval)
        {
            nvswitch_lib_disable_interrupts(nvswitch_dev->lib_device);
            return IRQ_WAKE_THREAD;
        }

        // PCI errors are handled else where.
        if (-NVL_PCI_ERROR == retval)
        {
            return IRQ_NONE;
        }

        if (NVL_SUCCESS != retval)
        {
            pr_err("nvidia-nvswitch: unrecoverable error in ISR\n");
            NVSWITCH_OS_ASSERT(0);
        }
        return IRQ_NONE;
    }

    pr_err("nvidia-nvswitch: unsupported IRQ mechanism in ISR\n");
    NVSWITCH_OS_ASSERT(0);

    return IRQ_NONE;
}

static irqreturn_t
nvswitch_isr_thread
(
    int   irq,
    void *arg
)
{
    NVSWITCH_DEV *nvswitch_dev = (NVSWITCH_DEV *)arg;
    NvlStatus retval;

    mutex_lock(&nvswitch_dev->device_mutex);

    retval = nvswitch_lib_service_interrupts(nvswitch_dev->lib_device);

    wake_up(&nvswitch_dev->wait_q_errors);

    if (nvswitch_dev->irq_mechanism == NVSWITCH_IRQ_PIN)
    {
        nvswitch_lib_enable_interrupts(nvswitch_dev->lib_device);
    }

    mutex_unlock(&nvswitch_dev->device_mutex);

    if (WARN_ON(retval != NVL_SUCCESS))
    {
        printk(KERN_ERR "%s: Interrupts disabled to avoid a storm\n",
               nvswitch_dev->name);
    }

    return IRQ_HANDLED;
}

static void
nvswitch_task_dispatch
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    NvU64 nsec;
    NvU64 timeout;
    NvS64 rc;

    if (NV_ATOMIC_READ(nvswitch_dev->task_q_ready) == 0)
    {
        return;
    }

    mutex_lock(&nvswitch_dev->device_mutex);

    nsec = nvswitch_lib_deferred_task_dispatcher(nvswitch_dev->lib_device);

    mutex_unlock(&nvswitch_dev->device_mutex);

    timeout = usecs_to_jiffies(nsec / NSEC_PER_USEC);

    rc = wait_event_interruptible_timeout(nvswitch_dev->wait_q_shutdown,
                              (NV_ATOMIC_READ(nvswitch_dev->task_q_ready) == 0),
                              timeout);

    //
    // These background tasks should rarely, if ever, get interrupted. We use
    // the "interruptible" variant of wait_event in order to avoid contributing
    // to the system load average (/proc/loadavg), and to avoid softlockup
    // warnings that can occur if a kernel thread lingers too long in an
    // uninterruptible state. If this does get interrupted, we'd like to debug
    // and find out why, so WARN in that case.
    //
    WARN_ON(rc < 0);

    //
    // Schedule a work item only if the above actually timed out or got
    // interrupted, without the condition becoming true.
    //
    if (rc <= 0)
    {
        if (!nv_kthread_q_schedule_q_item(&nvswitch_dev->task_q,
                                          &nvswitch_dev->task_item))
        {
            printk(KERN_ERR "%s: Failed to re-schedule background task\n",
                   nvswitch_dev->name);
        }
    }
}

static int
nvswitch_probe
(
    struct pci_dev *pci_dev,
    const struct pci_device_id *id_table
)
{
    NVSWITCH_DEV *nvswitch_dev = NULL;
    int rc = 0;
    int minor;

    if (!nvswitch_lib_validate_device_id(pci_dev->device))
    {
        return -EINVAL;
    }

    printk(KERN_INFO "nvidia-nvswitch: Probing device %04x:%02x:%02x.%x, "
           "Vendor Id = 0x%x, Device Id = 0x%x, Class = 0x%x \n",
           NV_PCI_DOMAIN_NUMBER(pci_dev),
           NV_PCI_BUS_NUMBER(pci_dev),
           NV_PCI_SLOT_NUMBER(pci_dev),
           PCI_FUNC(pci_dev->devfn),
           pci_dev->vendor,
           pci_dev->device,
           pci_dev->class);

    mutex_lock(&nvswitch.driver_mutex);

    minor = nvswitch_find_minor();
    if (minor >= NVSWITCH_DEVICE_INSTANCE_MAX)
    {
        rc = -ERANGE;
        goto find_minor_failed;
    }

    nvswitch_dev = kzalloc(sizeof(*nvswitch_dev), GFP_KERNEL);
    if (NULL == nvswitch_dev)
    {
        rc = -ENOMEM;
        goto kzalloc_failed;
    }

    mutex_init(&nvswitch_dev->device_mutex);
    init_waitqueue_head(&nvswitch_dev->wait_q_errors);
    init_waitqueue_head(&nvswitch_dev->wait_q_shutdown);

    snprintf(nvswitch_dev->name, sizeof(nvswitch_dev->name),
        NVSWITCH_DRIVER_NAME "%d", minor);

    snprintf(nvswitch_dev->sname, sizeof(nvswitch_dev->sname),
        NVSWITCH_SHORT_NAME "%d", minor);

    rc = pci_enable_device(pci_dev);
    if (rc)
    {
        printk(KERN_ERR "%s: Failed to enable PCI device : %d\n",
               nvswitch_dev->name,
               rc);
        goto pci_enable_device_failed;
    }

    pci_set_master(pci_dev);

    rc = pci_request_regions(pci_dev, nvswitch_dev->name);
    if (rc)
    {
        printk(KERN_ERR "%s: Failed to request memory regions : %d\n",
               nvswitch_dev->name,
               rc);
        goto pci_request_regions_failed;
    }

    nvswitch_dev->bar0 = pci_iomap(pci_dev, 0, 0);
    if (!nvswitch_dev->bar0)
    {
        rc = -ENOMEM;
        printk(KERN_ERR "%s: Failed to map BAR0 region : %d\n",
               nvswitch_dev->name,
               rc);
        goto pci_iomap_failed;
    }

    nvswitch_dev->pci_dev = pci_dev;
    nvswitch_dev->minor = minor;

    rc = nvswitch_init_device(nvswitch_dev);
    if (rc)
    {
        printk(KERN_ERR "%s: Failed to initialize device : %d\n",
               nvswitch_dev->name,
               rc);
        goto init_device_failed;
    }

    if (nvswitch_is_device_blacklisted(nvswitch_dev))
    {
        nvswitch_post_init_blacklisted(nvswitch_dev);
        goto blacklisted;
    }

    //
    // device_mutex held here because post_init entries may call soeService_HAL()
    // with IRQs on. see bug 2856314 for more info
    //
    mutex_lock(&nvswitch_dev->device_mutex);
    rc = nvswitch_post_init_device(nvswitch_dev);
    mutex_unlock(&nvswitch_dev->device_mutex);
    if (rc)
    {
        printk(KERN_ERR "%s:Failed during device post init : %d\n",
               nvswitch_dev->name, rc);
        goto post_init_device_failed;
    }

blacklisted:
    rc = nvswitch_init_background_tasks(nvswitch_dev);
    if (rc)
    {
        printk(KERN_ERR "%s: Failed to initialize background tasks : %d\n",
               nvswitch_dev->name,
               rc);
        goto init_background_task_failed;
    }

    pci_set_drvdata(pci_dev, nvswitch_dev);

    nvswitch_procfs_device_add(nvswitch_dev);

    list_add_tail(&nvswitch_dev->list_node, &nvswitch.devices);

    NV_ATOMIC_INC(nvswitch.count);

    mutex_unlock(&nvswitch.driver_mutex);

    return 0;

init_background_task_failed:
post_init_device_failed:
    nvswitch_deinit_device(nvswitch_dev);

init_device_failed:
    pci_iounmap(pci_dev, nvswitch_dev->bar0);

pci_iomap_failed:
    pci_release_regions(pci_dev);

pci_request_regions_failed:
#ifdef CONFIG_PCI
    pci_clear_master(pci_dev);
#endif
    pci_disable_device(pci_dev);

pci_enable_device_failed:
    kfree(nvswitch_dev);

kzalloc_failed:
find_minor_failed:
    mutex_unlock(&nvswitch.driver_mutex);

    return rc;
}

void
nvswitch_remove
(
    struct pci_dev *pci_dev
)
{
    NVSWITCH_DEV *nvswitch_dev;

    mutex_lock(&nvswitch.driver_mutex);

    nvswitch_dev = pci_get_drvdata(pci_dev);

    if (nvswitch_dev == NULL)
    {
        goto done;
    }

    printk(KERN_INFO "%s: removing device %04x:%02x:%02x.%x\n",
           nvswitch_dev->name,
           NV_PCI_DOMAIN_NUMBER(pci_dev),
           NV_PCI_BUS_NUMBER(pci_dev),
           NV_PCI_SLOT_NUMBER(pci_dev),
           PCI_FUNC(pci_dev->devfn));

    //
    // Synchronize with device operations such as .ioctls/.poll, and then mark
    // the device unusable.
    //
    mutex_lock(&nvswitch_dev->device_mutex);
    nvswitch_dev->unusable = NV_TRUE;
    mutex_unlock(&nvswitch_dev->device_mutex);

    NV_ATOMIC_DEC(nvswitch.count);

    list_del(&nvswitch_dev->list_node);

    nvswitch_deinit_i2c_adapters(nvswitch_dev);

    WARN_ON(!list_empty(&nvswitch_dev->i2c_adapter_list));

    pci_set_drvdata(pci_dev, NULL);

    nvswitch_deinit_background_tasks(nvswitch_dev);

    nvswitch_deinit_device(nvswitch_dev);

    pci_iounmap(pci_dev, nvswitch_dev->bar0);

    pci_release_regions(pci_dev);

#ifdef CONFIG_PCI
    pci_clear_master(pci_dev);
#endif

    pci_disable_device(pci_dev);

    nvswitch_procfs_device_remove(nvswitch_dev);

    // Free nvswitch_dev only if it is not in use.
    if (NV_ATOMIC_READ(nvswitch_dev->ref_count) == 0)
    {
        kfree(nvswitch_dev);
    }

done:
    mutex_unlock(&nvswitch.driver_mutex);

    return;
}

static void
nvswitch_load_bar_info
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    struct pci_dev *pci_dev = nvswitch_dev->pci_dev;
    nvlink_pci_info *info;
    NvU32 bar = 0;

    nvswitch_lib_get_device_info(nvswitch_dev->lib_device, &info);

    info->bars[0].offset = NVRM_PCICFG_BAR_OFFSET(0);
    pci_read_config_dword(pci_dev, info->bars[0].offset, &bar);

    info->bars[0].busAddress = (bar & PCI_BASE_ADDRESS_MEM_MASK);
    if (NV_PCI_RESOURCE_FLAGS(pci_dev, 0) & PCI_BASE_ADDRESS_MEM_TYPE_64)
    {
        pci_read_config_dword(pci_dev, info->bars[0].offset + 4, &bar);
        info->bars[0].busAddress |= (((NvU64)bar) << 32);
    }

    info->bars[0].baseAddr = NV_PCI_RESOURCE_START(pci_dev, 0);

    info->bars[0].barSize = NV_PCI_RESOURCE_SIZE(pci_dev, 0);

    info->bars[0].pBar = nvswitch_dev->bar0;
}

static int
_nvswitch_initialize_msix_interrupt
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    // Not supported (bug 3018806)
    return -EINVAL;
}

static int
_nvswitch_initialize_msi_interrupt
(
    NVSWITCH_DEV *nvswitch_dev
)
{
#ifdef CONFIG_PCI_MSI
    struct pci_dev *pci_dev = nvswitch_dev->pci_dev;
    int rc;

    rc = pci_enable_msi(pci_dev);
    if (rc)
    {
        return rc;
    }

    return 0;
#else
    return -EINVAL;
#endif
}

static int
_nvswitch_get_irq_caps(NVSWITCH_DEV *nvswitch_dev, unsigned long *irq_caps)
{
    struct pci_dev *pci_dev;

    if (!nvswitch_dev || !irq_caps)
        return -EINVAL;

    pci_dev = nvswitch_dev->pci_dev;

    if (pci_find_capability(pci_dev, PCI_CAP_ID_MSIX))
        set_bit(NVSWITCH_IRQ_MSIX, irq_caps);

    if (pci_find_capability(pci_dev, PCI_CAP_ID_MSI))
        set_bit(NVSWITCH_IRQ_MSI, irq_caps);

    if (nvswitch_lib_use_pin_irq(nvswitch_dev->lib_device))
        set_bit(NVSWITCH_IRQ_PIN, irq_caps);

    return 0;
}

static int
nvswitch_initialize_device_interrupt
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    struct pci_dev *pci_dev = nvswitch_dev->pci_dev;
    int flags = 0;
    unsigned long irq_caps = 0;
    int rc;

    if (_nvswitch_get_irq_caps(nvswitch_dev, &irq_caps))
    {
        pr_err("%s: failed to retrieve device interrupt capabilities\n",
               nvswitch_dev->name);
        return -EINVAL;
    }

    nvswitch_dev->irq_mechanism = NVSWITCH_IRQ_NONE;

    if (test_bit(NVSWITCH_IRQ_MSIX, &irq_caps))
    {
        rc = _nvswitch_initialize_msix_interrupt(nvswitch_dev);
        if (!rc)
        {
            nvswitch_dev->irq_mechanism = NVSWITCH_IRQ_MSIX;
            pr_info("%s: using MSI-X\n", nvswitch_dev->name);
        }
    }

    if (nvswitch_dev->irq_mechanism == NVSWITCH_IRQ_NONE
        && test_bit(NVSWITCH_IRQ_MSI, &irq_caps))
    {
        rc = _nvswitch_initialize_msi_interrupt(nvswitch_dev);
        if (!rc)
        {
            nvswitch_dev->irq_mechanism = NVSWITCH_IRQ_MSI;
            pr_info("%s: using MSI\n", nvswitch_dev->name);
        }
    }

    if (nvswitch_dev->irq_mechanism == NVSWITCH_IRQ_NONE
        && test_bit(NVSWITCH_IRQ_PIN, &irq_caps))
    {
        flags |= IRQF_SHARED;
        nvswitch_dev->irq_mechanism = NVSWITCH_IRQ_PIN;
        pr_info("%s: using PCI pin\n", nvswitch_dev->name);
    }

    if (nvswitch_dev->irq_mechanism == NVSWITCH_IRQ_NONE)
    {
        pr_err("%s: No supported interrupt mechanism was found. This device supports:\n",
               nvswitch_dev->name);

        if (test_bit(NVSWITCH_IRQ_MSIX, &irq_caps))
            pr_err("%s: MSI-X\n", nvswitch_dev->name);
        if (test_bit(NVSWITCH_IRQ_MSI, &irq_caps))
            pr_err("%s: MSI\n", nvswitch_dev->name);
        if (test_bit(NVSWITCH_IRQ_PIN, &irq_caps))
             pr_err("%s: PCI Pin\n", nvswitch_dev->name);

        return -EINVAL;
    }

    rc = request_threaded_irq(pci_dev->irq,
                              nvswitch_isr_pending,
                              nvswitch_isr_thread,
                              flags, nvswitch_dev->sname,
                              nvswitch_dev);
    if (rc)
    {
#ifdef CONFIG_PCI_MSI
        if (nvswitch_dev->irq_mechanism == NVSWITCH_IRQ_MSI)
        {
            pci_disable_msi(pci_dev);
        }
#endif
        printk(KERN_ERR "%s: failed to get IRQ\n",
               nvswitch_dev->name);

        return rc;
    }

    return 0;
}

void
nvswitch_shutdown_device_interrupt
(
    NVSWITCH_DEV *nvswitch_dev
)
{
    struct pci_dev *pci_dev = nvswitch_dev->pci_dev;

    free_irq(pci_dev->irq, nvswitch_dev);
#ifdef CONFIG_PCI_MSI
    if (nvswitch_dev->irq_mechanism == NVSWITCH_IRQ_MSI)
    {
        pci_disable_msi(pci_dev);
    }
#endif
}

static void
nvswitch_ctl_exit
(
    void
)
{
    cdev_del(&nvswitch.cdev_ctl);
}

static int
nvswitch_ctl_init
(
    int major
)
{
    int rc = 0;
    dev_t nvswitch_ctl = MKDEV(major, NVSWITCH_CTL_MINOR);

    cdev_init(&nvswitch.cdev_ctl, &ctl_fops);

    nvswitch.cdev_ctl.owner = THIS_MODULE;

    rc = cdev_add(&nvswitch.cdev_ctl, nvswitch_ctl, 1);
    if (rc < 0)
    {
        printk(KERN_ERR "nvidia-nvswitch: Unable to create cdev ctl\n");
        return rc;
    }

    return 0;
}

//
// Initialize nvswitch driver SW state.  This is currently called
// from the RM as a backdoor interface, and not by the Linux device
// manager
//
int
nvswitch_init
(
    void
)
{
    int rc;

    if (nvswitch.initialized)
    {
        printk(KERN_ERR "nvidia-nvswitch: Interface already initialized\n");
        return -EBUSY;
    }

    BUILD_BUG_ON(NVSWITCH_DEVICE_INSTANCE_MAX >= NVSWITCH_MINOR_COUNT);

    mutex_init(&nvswitch.driver_mutex);

    INIT_LIST_HEAD(&nvswitch.devices);

    rc = alloc_chrdev_region(&nvswitch.devno,
                             0,
                             NVSWITCH_MINOR_COUNT,
                             NVSWITCH_DRIVER_NAME);
    if (rc < 0)
    {
        printk(KERN_ERR "nvidia-nvswitch: Unable to create cdev region\n");
        goto alloc_chrdev_region_fail;
    }

    printk(KERN_ERR, "nvidia-nvswitch: Major: %d Minor: %d\n",
           MAJOR(nvswitch.devno),
           MINOR(nvswitch.devno));

    cdev_init(&nvswitch.cdev, &device_fops);
    nvswitch.cdev.owner = THIS_MODULE;
    rc = cdev_add(&nvswitch.cdev, nvswitch.devno, NVSWITCH_DEVICE_INSTANCE_MAX);
    if (rc < 0)
    {
        printk(KERN_ERR "nvidia-nvswitch: Unable to create cdev\n");
        goto cdev_add_fail;
    }

    rc = nvswitch_procfs_init();
    if (rc < 0)
    {
        goto nvswitch_procfs_init_fail;
    }

    rc = pci_register_driver(&nvswitch_pci_driver);
    if (rc < 0)
    {
        printk(KERN_ERR "nvidia-nvswitch: Failed to register driver : %d\n", rc);
        goto pci_register_driver_fail;
    }

    rc = nvswitch_ctl_init(MAJOR(nvswitch.devno));
    if (rc < 0)
    {
        goto nvswitch_ctl_init_fail;
    }

    nvswitch.initialized = NV_TRUE;

    return 0;

nvswitch_ctl_init_fail:
    pci_unregister_driver(&nvswitch_pci_driver);

pci_register_driver_fail:
nvswitch_procfs_init_fail:
    cdev_del(&nvswitch.cdev);

cdev_add_fail:
    unregister_chrdev_region(nvswitch.devno, NVSWITCH_MINOR_COUNT);

alloc_chrdev_region_fail:

    return rc;
}

//
// Clean up driver state on exit.  Currently called from RM backdoor call,
// and not by the Linux device manager.
//
void
nvswitch_exit
(
    void
)
{
    if (NV_FALSE == nvswitch.initialized)
    {
        return;
    }

    nvswitch_ctl_exit();

    pci_unregister_driver(&nvswitch_pci_driver);

    nvswitch_procfs_exit();

    cdev_del(&nvswitch.cdev);

    unregister_chrdev_region(nvswitch.devno, NVSWITCH_MINOR_COUNT);

    WARN_ON(!list_empty(&nvswitch.devices));

    nvswitch.initialized = NV_FALSE;
}

//
// Get current time in seconds.nanoseconds
// In this implementation, the time is from epoch time
// (midnight UTC of January 1, 1970)
//
NvU64
nvswitch_os_get_platform_time
(
    void
)
{
    struct timespec64 ts;

    ktime_get_raw_ts64(&ts);
    return (NvU64) timespec64_to_ns(&ts);
}

void
nvswitch_os_print
(
    const int  log_level,
    const char *fmt,
    ...
)
{
    va_list arglist;
    char   *kern_level;
    char    fmt_printk[NVSWITCH_LOG_BUFFER_SIZE];

    switch (log_level)
    {
        case NVSWITCH_DBG_LEVEL_MMIO:
            kern_level = KERN_DEBUG;
            break;
        case NVSWITCH_DBG_LEVEL_INFO:
            kern_level = KERN_INFO;
            break;
        case NVSWITCH_DBG_LEVEL_SETUP:
            kern_level = KERN_INFO;
            break;
        case NVSWITCH_DBG_LEVEL_WARN:
            kern_level = KERN_WARNING;
            break;
        case NVSWITCH_DBG_LEVEL_ERROR:
            kern_level = KERN_ERR;
            break;
        default:
            kern_level = KERN_DEFAULT;
            break;
    }

    va_start(arglist, fmt);
    snprintf(fmt_printk, sizeof(fmt_printk), "%s%s", kern_level, fmt);
    vprintk(fmt_printk, arglist);
    va_end(arglist);
}

void
nvswitch_os_override_platform
(
    void *os_handle,
    NvBool *rtlsim
)
{
}

NvlStatus
nvswitch_os_read_registery_binary
(
    void *os_handle,
    const char *name,
    NvU8 *data,
    NvU32 length
)
{
    return -NVL_ERR_NOT_SUPPORTED;
}

NvU32
nvswitch_os_get_device_count
(
    void
)
{
    return NV_ATOMIC_READ(nvswitch.count);
}

//
// A helper to convert a string to an unsigned int.
//
// The string should be NULL terminated.
// Only works with base16 values.
//
static int
nvswitch_os_strtouint
(
    char *str,
    unsigned int *data
)
{
    char *p;
    unsigned long long val;

    if (!str || !data)
    {
        return -EINVAL;
    }

    *data = 0;
    val = 0;
    p = str;

    while (*p != '\0')
    {
        if ((tolower(*p) == 'x') && (*str == '0') && (p == str + 1))
        {
            p++;
        }
        else if (*p >='0' && *p <= '9')
        {
            val = val * 16 + (*p - '0');
            p++;
        }
        else if (tolower(*p) >= 'a' && tolower(*p) <= 'f')
        {
            val = val * 16 + (tolower(*p) - 'a' + 10);
            p++;
        }
        else
        {
            return -EINVAL;
        }
    }

    if (val > 0xFFFFFFFF)
    {
        return -EINVAL;
    }

    *data = (unsigned int)val;

    return 0;
}

NvlStatus
nvswitch_os_read_registry_dword
(
    void *os_handle,
    const char *name,
    NvU32 *data
)
{
    char *regkey, *regkey_val_start, *regkey_val_end;
    char regkey_val[NVSWITCH_REGKEY_VALUE_LEN + 1];
    NvU32 regkey_val_len = 0;

    *data = 0;

    if (!NvSwitchRegDwords)
    {
        return -NVL_ERR_GENERIC;
    }

    regkey = strstr(NvSwitchRegDwords, name);
    if (!regkey)
    {
        return -NVL_ERR_GENERIC;
    }

    regkey = strchr(regkey, '=');
    if (!regkey)
    {
        return -NVL_ERR_GENERIC;
    }

    regkey_val_start = regkey + 1;

    regkey_val_end = strchr(regkey, ';');
    if (!regkey_val_end)
    {
        regkey_val_end = strchr(regkey, '\0');
    }

    regkey_val_len = regkey_val_end - regkey_val_start;
    if (regkey_val_len > NVSWITCH_REGKEY_VALUE_LEN || regkey_val_len == 0)
    {
        return -NVL_ERR_GENERIC;
    }

    strncpy(regkey_val, regkey_val_start, regkey_val_len);
    regkey_val[regkey_val_len] = '\0';

    if (nvswitch_os_strtouint(regkey_val, data) != 0)
    {
        return -NVL_ERR_GENERIC;
    }

    return NVL_SUCCESS;
}

static NvBool
_nvswitch_is_space(const char ch)
{
    return ((ch == ' ') || ((ch >= '\t') && (ch <= '\r')));
}

static char *
_nvswitch_remove_spaces(const char *in)
{
    unsigned int len = nvswitch_os_strlen(in) + 1;
    const char *in_ptr;
    char *out, *out_ptr;

    out = nvswitch_os_malloc(len);
    if (out == NULL)
        return NULL;

    in_ptr = in;
    out_ptr = out;

    while (*in_ptr != '\0')
    {
        if (!_nvswitch_is_space(*in_ptr))
            *out_ptr++ = *in_ptr;
        in_ptr++;
    }
    *out_ptr = '\0';

    return out;
}

/*
 * Compare given string UUID with the NvSwitchBlacklist registry parameter string and
 * return whether the UUID is in the NvSwitch blacklist
 */
NvBool
nvswitch_os_is_uuid_in_blacklist
(
    NvUuid *uuid
)
{
    char *list;
    char *ptr;
    char *token;
    NvU8 uuid_string[NVSWITCH_UUID_STRING_LENGTH];

    if (NvSwitchBlacklist == NULL)
        return NV_FALSE;

    if (nvswitch_uuid_to_string(uuid, uuid_string, NVSWITCH_UUID_STRING_LENGTH) == 0)
        return NV_FALSE;

    if ((list = _nvswitch_remove_spaces(NvSwitchBlacklist)) == NULL)
        return NV_FALSE;

    ptr = list;

    while ((token = strsep(&ptr, ",")) != NULL)
    {
        if (strcmp(token, uuid_string) == 0)
        {
            nvswitch_os_free(list);
            return NV_TRUE;
        }
    }
    nvswitch_os_free(list);
    return NV_FALSE;
}


NvlStatus
nvswitch_os_alloc_contig_memory
(
    void *os_handle,
    void **virt_addr,
    NvU32 size,
    NvBool force_dma32
)
{
    NvU32 gfp_flags;
    unsigned long nv_gfp_addr = 0;

    if (!virt_addr)
        return -NVL_BAD_ARGS;

    gfp_flags = GFP_KERNEL | (force_dma32 ? GFP_DMA32 : 0);
    NV_GET_FREE_PAGES(nv_gfp_addr, get_order(size), gfp_flags);

    if(!nv_gfp_addr)
    {
        pr_err("nvidia-nvswitch: unable to allocate kernel memory\n");
        return -NVL_NO_MEM;
    }

    *virt_addr = (void *)nv_gfp_addr;

    return NVL_SUCCESS;
}

void
nvswitch_os_free_contig_memory
(
    void *os_handle,
    void *virt_addr,
    NvU32 size
)
{
    NV_FREE_PAGES((unsigned long)virt_addr, get_order(size));
}

static inline int
_nvswitch_to_pci_dma_direction
(
    NvU32 direction
)
{
    if (direction == NVSWITCH_DMA_DIR_TO_SYSMEM)
        return DMA_FROM_DEVICE;
    else if (direction == NVSWITCH_DMA_DIR_FROM_SYSMEM)
        return DMA_TO_DEVICE;
    else
        return DMA_BIDIRECTIONAL;
}

NvlStatus
nvswitch_os_map_dma_region
(
    void *os_handle,
    void *cpu_addr,
    NvU64 *dma_handle,
    NvU32 size,
    NvU32 direction
)
{
    int dma_dir;
    struct pci_dev *pdev = (struct pci_dev *)os_handle;

    if (!pdev || !cpu_addr || !dma_handle)
        return -NVL_BAD_ARGS;

    dma_dir = _nvswitch_to_pci_dma_direction(direction);

    *dma_handle = (NvU64)dma_map_single(&pdev->dev, cpu_addr, size, dma_dir);

    if (dma_mapping_error(&pdev->dev, *dma_handle))
    {
        pr_err("nvidia-nvswitch: unable to create PCI DMA mapping\n");
        return -NVL_ERR_GENERIC;
    }

    return NVL_SUCCESS;
}

NvlStatus
nvswitch_os_unmap_dma_region
(
    void *os_handle,
    void *cpu_addr,
    NvU64 dma_handle,
    NvU32 size,
    NvU32 direction
)
{
    int dma_dir;
    struct pci_dev *pdev = (struct pci_dev *)os_handle;

    if (!pdev || !cpu_addr)
        return -NVL_BAD_ARGS;

    dma_dir = _nvswitch_to_pci_dma_direction(direction);

    dma_unmap_single(&pdev->dev, dma_handle, size, dma_dir);

    return NVL_SUCCESS;
}

NvlStatus
nvswitch_os_set_dma_mask
(
    void *os_handle,
    NvU32 dma_addr_width
)
{
    struct pci_dev *pdev = (struct pci_dev *)os_handle;

    if (!pdev)
        return -NVL_BAD_ARGS;

    if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(dma_addr_width)))
        return -NVL_ERR_GENERIC;

    return NVL_SUCCESS;
}

NvlStatus
nvswitch_os_sync_dma_region_for_cpu
(
    void *os_handle,
    NvU64 dma_handle,
    NvU32 size,
    NvU32 direction
)
{
    int dma_dir;
    struct pci_dev *pdev = (struct pci_dev *)os_handle;

    if (!pdev)
        return -NVL_BAD_ARGS;

    dma_dir = _nvswitch_to_pci_dma_direction(direction);

    dma_sync_single_for_cpu(&pdev->dev, dma_handle, size, dma_dir);

    return NVL_SUCCESS;
}

NvlStatus
nvswitch_os_sync_dma_region_for_device
(
    void *os_handle,
    NvU64 dma_handle,
    NvU32 size,
    NvU32 direction
)
{
    int dma_dir;
    struct pci_dev *pdev = (struct pci_dev *)os_handle;

    if (!pdev)
        return -NVL_BAD_ARGS;

    dma_dir = _nvswitch_to_pci_dma_direction(direction);

    dma_sync_single_for_device(&pdev->dev, dma_handle, size, dma_dir);

    return NVL_SUCCESS;
}

static inline void *
_nvswitch_os_malloc
(
    NvLength size
)
{
    void *ptr = NULL;

    if (!NV_MAY_SLEEP())
    {
        if (size <= NVSWITCH_KMALLOC_LIMIT)
        {
            ptr = kmalloc(size, NV_GFP_ATOMIC);
        }
    }
    else
    {
        if (size <= NVSWITCH_KMALLOC_LIMIT)
        {
            ptr = kmalloc(size, NV_GFP_NO_OOM);
        }

        if (ptr == NULL)
        {
            ptr = vmalloc(size);
        }
    }

    return ptr;
}

void *
nvswitch_os_malloc_trace
(
    NvLength size,
    const char *file,
    NvU32 line
)
{
#if defined(NV_MEM_LOGGER)
    void *ptr = _nvswitch_os_malloc(size);
    if (ptr)
    {
        nv_memdbg_add(ptr, size, file, line);
    }

    return ptr;
#else
    return _nvswitch_os_malloc(size);
#endif
}

static inline void
_nvswitch_os_free
(
    void *ptr
)
{
    if (!ptr)
        return;

    if (is_vmalloc_addr(ptr))
    {
        vfree(ptr);
    }
    else
    {
        kfree(ptr);
    }
}

void
nvswitch_os_free
(
    void *ptr
)
{
#if defined (NV_MEM_LOGGER)
    if (ptr == NULL)
        return;

    nv_memdbg_remove(ptr, 0, NULL, 0);

    return _nvswitch_os_free(ptr);
#else
    return _nvswitch_os_free(ptr);
#endif
}

NvLength
nvswitch_os_strlen
(
    const char *str
)
{
    return strlen(str);
}

char*
nvswitch_os_strncpy
(
    char *dest,
    const char *src,
    NvLength length
)
{
    return strncpy(dest, src, length);
}

int
nvswitch_os_strncmp
(
    const char *s1,
    const char *s2,
    NvLength length
)
{
    return strncmp(s1, s2, length);
}

void *
nvswitch_os_memset
(
    void *dest,
    int value,
    NvLength size
)
{
     return memset(dest, value, size);
}

void *
nvswitch_os_memcpy
(
    void *dest,
    const void *src,
    NvLength size
)
{
    return memcpy(dest, src, size);
}

int
nvswitch_os_memcmp
(
    const void *s1,
    const void *s2,
    NvLength size
)
{
    return memcmp(s1, s2, size);
}

NvU32
nvswitch_os_mem_read32
(
    const volatile void * address
)
{
    return (*(const volatile NvU32*)(address));
}

void
nvswitch_os_mem_write32
(
    volatile void *address,
    NvU32 data
)
{
    (*(volatile NvU32 *)(address)) = data;
}

NvU64
nvswitch_os_mem_read64
(
    const volatile void * address
)
{
    return (*(const volatile NvU64 *)(address));
}

void
nvswitch_os_mem_write64
(
    volatile void *address,
    NvU64 data
)
{
    (*(volatile NvU64 *)(address)) = data;
}

int
nvswitch_os_snprintf
(
    char *dest,
    NvLength size,
    const char *fmt,
    ...
)
{
    va_list arglist;
    int chars_written;

    va_start(arglist, fmt);
    chars_written = vsnprintf(dest, size, fmt, arglist);
    va_end(arglist);

    return chars_written;
}

int
nvswitch_os_vsnprintf
(
    char *buf,
    NvLength size,
    const char *fmt,
    va_list arglist
)
{
    return vsnprintf(buf, size, fmt, arglist);
}

void
nvswitch_os_assert_log
(
    int cond,
    const char *fmt,
    ...
)
{
    if(cond == 0x0)
    {
        if (printk_ratelimit())
        {
            va_list arglist;
            char fmt_printk[NVSWITCH_LOG_BUFFER_SIZE];

            va_start(arglist, fmt);
            vsnprintf(fmt_printk, sizeof(fmt_printk), fmt, arglist);
            va_end(arglist);
            nvswitch_os_print(NVSWITCH_DBG_LEVEL_ERROR, fmt_printk);
            WARN_ON(1);
         }
         dbg_breakpoint();
    }
}

/*
 * Sleep for specified milliseconds. Yields the CPU to scheduler.
 */
void
nvswitch_os_sleep
(
    unsigned int ms
)
{
    NV_STATUS status;
    status = nv_sleep_ms(ms);

    if (status != NV_OK)
    {
        if (printk_ratelimit())
        {
            nvswitch_os_print(NVSWITCH_DBG_LEVEL_ERROR, "NVSwitch: requested"
                              " sleep duration %d msec exceeded %d msec\n",
                              ms, NV_MAX_ISR_DELAY_MS);
            WARN_ON(1);
        }
    }
}

NvlStatus
nvswitch_os_acquire_fabric_mgmt_cap
(
    void *osPrivate,
    NvU64 capDescriptor
)
{
    int dup_fd = -1;
    nvswitch_file_private_t *private_data = (nvswitch_file_private_t *)osPrivate;

    if (private_data == NULL)
    {
        return -NVL_BAD_ARGS;
    }

    dup_fd = nvlink_cap_acquire((int)capDescriptor,
                                NVLINK_CAP_FABRIC_MANAGEMENT);
    if (dup_fd < 0)
    {
        return -NVL_ERR_OPERATING_SYSTEM;
    }

    private_data->capability_fds.fabric_mgmt = dup_fd;
    return NVL_SUCCESS;
}

int
nvswitch_os_is_fabric_manager
(
    void *osPrivate
)
{
    nvswitch_file_private_t *private_data = (nvswitch_file_private_t *)osPrivate;

    /* Make sure that fabric mgmt capbaility fd is valid */
    if ((private_data == NULL) ||
        (private_data->capability_fds.fabric_mgmt < 0))
    {
        return 0;
    }

    return 1;
}

int
nvswitch_os_is_admin
(
    void
)
{
    return NV_IS_SUSER();
}

#define NV_KERNEL_RELEASE    ((LINUX_VERSION_CODE >> 16) & 0x0ff)
#define NV_KERNEL_VERSION    ((LINUX_VERSION_CODE >> 8)  & 0x0ff)
#define NV_KERNEL_SUBVERSION ((LINUX_VERSION_CODE)       & 0x0ff)

NvlStatus
nvswitch_os_get_os_version
(
    NvU32 *pMajorVer,
    NvU32 *pMinorVer,
    NvU32 *pBuildNum
)
{
    if (pMajorVer)
        *pMajorVer = NV_KERNEL_RELEASE;
    if (pMinorVer)
        *pMinorVer = NV_KERNEL_VERSION;
    if (pBuildNum)
        *pBuildNum = NV_KERNEL_SUBVERSION;

    return NVL_SUCCESS;
}

/*!
 * @brief: OS specific handling to add an event.
 */
NvlStatus
nvswitch_os_add_client_event
(
    void            *osHandle,
    void            *osPrivate,
    NvU32           eventId
)
{
    return NVL_SUCCESS;
}

/*!
 * @brief: OS specific handling to remove all events corresponding to osPrivate.
 */
NvlStatus
nvswitch_os_remove_client_event
(
    void            *osHandle,
    void            *osPrivate
)
{
    return NVL_SUCCESS;
}

/*!
 * @brief: OS specific handling to notify an event.
 */
NvlStatus
nvswitch_os_notify_client_event
(
    void *osHandle,
    void *osPrivate,
    NvU32 eventId
)
{
    nvswitch_file_private_t *private_data = (nvswitch_file_private_t *)osPrivate;

    if (private_data == NULL)
    {
        return -NVL_BAD_ARGS;
    }

    private_data->file_event.event_pending = NV_TRUE;
    wake_up_interruptible(&private_data->file_event.wait_q_event);

    return NVL_SUCCESS;
}

/*!
 * @brief: Gets OS specific support for the REGISTER_EVENTS ioctl
 */
NvlStatus
nvswitch_os_get_supported_register_events_params
(
    NvBool *many_events,
    NvBool *os_descriptor
)
{
    *many_events   = NV_FALSE;
    *os_descriptor = NV_FALSE;
    return NVL_SUCCESS;
}
