/* _NVRM_COPYRIGHT_BEGIN_
 *
 * Copyright 2001 by NVIDIA Corporation.  All rights reserved.  All
 * information contained herein is proprietary and confidential to NVIDIA
 * Corporation.  Any use, reproduction, or disclosure without the written
 * permission of NVIDIA Corporation is prohibited.
 *
 * _NVRM_COPYRIGHT_END_
 */

#ifndef __NV_FREEBSD_H__
#define __NV_FREEBSD_H__

#ifdef TRUE
#undef TRUE
#endif

#ifdef FALSE
#undef FALSE
#endif

#include <sys/param.h>

#if __FreeBSD_version >= 800000
#error This driver does not support FreeBSD 8.x/-CURRENT!
#endif
#if __FreeBSD_version >= 700000 && __FreeBSD_version < 700055
#error This driver does not support FreeBSD 7.x/-CURRENT!
#endif
#if __FreeBSD_version >= 600000 && __FreeBSD_version < 600034
#error This driver does not support FreeBSD 6.x/-CURRENT!
#endif
#if __FreeBSD_version < 503000
#error This driver requires FreeBSD 5.3 or later!
#endif

#include <sys/systm.h>
#include <sys/types.h>
#include <sys/queue.h>
#include <sys/pciio.h>
#include <sys/vnode.h>

#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/ioccom.h>
#include <sys/malloc.h>
#include <sys/socket.h>
#include <sys/sysent.h>
#include <sys/ctype.h>
#include <sys/sysctl.h>

#include <machine/cpu.h>
#include <machine/resource.h>
#include <machine/clock.h>
#include <machine/stdarg.h>
#include <machine/bus.h>
#include <machine/specialreg.h>

#include <sys/conf.h>
#include <sys/rman.h>
#include <sys/proc.h>
#include <sys/lock.h>
#include <sys/mman.h>
#include <sys/file.h>
#include <sys/poll.h>

#include <sys/syscall.h>
#include <sys/bus.h>
#include <sys/memrange.h>
#include <sys/sysproto.h>
#include <sys/signalvar.h>

#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/vm_extern.h>
#include <vm/vm_object.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/uma.h>

#if __FreeBSD_version < 800004
#include <pci/agpvar.h>
#else
#include <dev/agp/agpvar.h>
#endif
#include <sys/agpio.h>

#if defined(NVCPU_X86_64)
#define NV_MMAP_TO_VM_OFFSET(_off) ((_off) | 0xfffff00000000000)
#define NV_VM_TO_MMAP_OFFSET(_off) ((_off) & 0x00000fffffffffff)
#else
#define NV_MMAP_TO_VM_OFFSET(_off) ((_off) + VM_MIN_KERNEL_ADDRESS)
#define NV_VM_TO_MMAP_OFFSET(_off) ((_off) - VM_MIN_KERNEL_ADDRESS)
#endif

#include <sys/smp.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <sys/kdb.h>
#include <sys/filedesc.h>

#if __FreeBSD_version >= 700055
#include <sys/priv.h>
#endif

#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sx.h>
#include <sys/condvar.h>

#if defined(NVCPU_X86) && defined(PAE)
#error This driver does not support PAE enabled kernels!
#endif

#define CURTHREAD curthread

#if __FreeBSD_version >= 800049
#define suser(_td) priv_check((_td), PRIV_DRIVER)
#endif

/*
 * The resource manager client tracking needs an identifier that uniquely
 * represents a client connection across threads. It needs to be specific
 * to the thread that allocated the client. The Linux struct file pointer
 * is such an identifier. The per-process file descriptor table is enough
 * on FreeBSD, due to its thread-aware reference counting.
 */

#define __TD_FDT(_td) (void *)((_td)->td_proc->p_fd)

#if __FreeBSD_version >= 601100
#define __NV_ITHREAD() (curthread->td_pflags & TDP_ITHREAD)
#else
#define __NV_ITHREAD() (curthread->td_ithd != NULL)
#endif

/*
 * The NVIDIA kernel module's malloc identifier, needed for both tracking
 * and actual allocation/freeing purposes - declared here, but defined in
 * nvidia_os.c.
 */

MALLOC_DECLARE(M_NVIDIA);

RM_STATUS os_alloc_contig_pages(void **, U032);
void os_free_contig_pages(void *, U032);

/*
 * Enable/Disable support for FreeBSD's AGP GART driver. Please note that
 * agp.ko may need to be preloaded from loader.conf; this define does not
 * influence support for NVIDIA's builtin AGP GART driver.
 */

#undef NV_SUPPORT_OS_AGP

/*
 * This option decides if the driver will be built with support for Linux
 * or Linux 32-bit (FreeBSD/amd64) compatibility. This makes nvidia.ko
 * dependent on linux.ko; if you don't need Linux compatibility, then you
 * can safely unset this flag.
 */

#define NV_SUPPORT_LINUX_COMPAT

/*
 * Enable/Disable support for ACPI Power Management. This is untested and
 * thus disabled by default (this is 5.x/-CURRENT only).
 */

#undef NV_SUPPORT_ACPI_PM


/*
 * The DMA memory allocation tracking structure. DMA memory alloctions on
 * FreeBSD are tracked with their base address and size. Since PCI memory
 * is allocated from kernel virtual memory and since AGP allocations have
 * a contiguous range of addresses in the AGP aperture, base and size are
 * sufficient to track allocations.
 */

typedef
struct nvidia_alloc {
    SLIST_ENTRY(nvidia_alloc) list;
    uint32_t cache_type;
    u_int32_t size;
    vm_offset_t address;
    int alloc_type_contiguous;
    NvU64 *pte_array;
} nvidia_alloc_t;

typedef
struct nvidia_event {
    STAILQ_ENTRY(nvidia_event) queue;
    nv_event_t event;
} nvidia_event_t;

typedef
struct nvidia_filep {
    STAILQ_ENTRY(nvidia_filep) queue;
    void *fd_table;
    u_int32_t fd_refcnt;
} nvidia_filep_t;

typedef
struct nvidia_softc {
    device_t dev;
    device_t agp_dev;

    struct resource *BAR_recs[NV_GPU_NUM_BARS];
    int BAR_rids[NV_GPU_NUM_BARS];

    struct resource *irq;
    void *irq_ih;
    int   irq_rid;

    nv_stack_t *attach_sp;
    nv_stack_t *api_sp;
    nv_stack_t *pci_cfgchk_sp;
    nv_stack_t *isr_sp;
    nv_stack_t *timer_sp;

    struct resource *iop;
    int iop_rid;

    bus_space_handle_t bs_handle;
    bus_space_tag_t bs_tag;

    struct cdev *cdev;
    nv_state_t *nv_state;

    /* queue of OS events */
    STAILQ_HEAD(event_queue, nvidia_event) event_queue;

    /* queue of fd table references */
    STAILQ_HEAD(filep_queue, nvidia_filep) filep_queue;

    struct sysctl_ctx_list sysctl_ctx;
    struct selinfo rsel;

    struct callout_handle timer_ch;

    /* list of allocations */
    SLIST_HEAD(alloc_list, nvidia_alloc) alloc_list;

    uint32_t refcnt;

    struct mtx rm_mtx;
    struct sx api_sx;

} nvidia_softc_t;

#define CDEV_CTL_MINOR  255

extern devclass_t nvidia_devclass;

extern struct nvidia_softc nvidia_ctl_sc;
extern nv_state_t nvidia_ctl_state;

extern const char *pNVRM_ID;

#define PCIR_CAP_LIST_ID   0x00
#define PCIR_CAP_LIST_NEXT 0x01
#define PCIR_CAP_ID_AGP    0x02
#define PCIR_CAP_ID_EXP    0x10

#if !defined(PCIS_DISPLAY_3D)
#define PCIS_DISPLAY_3D    0x002
#endif
#if !defined(PCIM_CMD_INTXDIS)
#define PCIM_CMD_INTXDIS   0x400
#endif

#if !defined(PAT_UNCACHEABLE)
#define PAT_UNCACHEABLE         0x00
#endif
#if !defined(PAT_WRITE_COMBINING)
#define PAT_WRITE_COMBINING     0x01
#endif
#if !defined(PAT_WRITE_BACK)
#define PAT_WRITE_BACK          0x06
#endif

#if __FreeBSD_version < 602110 || __FreeBSD_version >= 700000 && __FreeBSD_version < 700055
static inline int pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
{
    vm_offset_t tmp;
    pt_entry_t *ptep;

    if (mode == PAT_UNCACHEABLE) {
        for (tmp = va; tmp < (va + size); tmp += PAGE_SIZE) {
            ptep = vtopte(tmp);
            pte_store(ptep, pte_load(ptep) | PG_N);
        }
        pmap_invalidate_range(kernel_pmap, va, tmp);
    }

    return 0;
}

static inline void *pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
{
    vm_offset_t tmp, va;
    uint32_t cache_bits;

    va = kmem_alloc_nofault(kernel_map, size);
    if (va != 0) {
        cache_bits = (mode != PAT_WRITE_BACK) ? PG_N : 0;
        for (tmp = va; tmp < (va + size); tmp += PAGE_SIZE) {
            pt_entry_t *ptep = vtopte(tmp);
            pte_store(ptep, pa | PG_RW | PG_V | PG_G | cache_bits);
            pa += PAGE_SIZE;
        }
        pmap_invalidate_range(kernel_pmap, va, tmp);
    }

    return (void *)va;
}
#endif

static inline void pmap_unmapdev_attr(vm_offset_t va, vm_size_t size)
{
#if __FreeBSD_version < 602110 || __FreeBSD_version >= 700000 && __FreeBSD_version < 700055
    vm_offset_t tmp;

    if (va != 0) {
        size = NV_ALIGN_UP(size, PAGE_SIZE);
        for (tmp = va; tmp < (va + size); tmp += PAGE_SIZE)
            pte_clear(vtopte(tmp));
        pmap_invalidate_range(kernel_pmap, va, tmp);
        kmem_free(kernel_map, va, size);
    }
#else
    pmap_unmapdev(va, size);
#endif
}

/*
 * Entries in the NVIDIA glue-layer registry are now described by the new
 * shared nv_parm_t structure; please review nvidia_os_registry.c in case
 * you need to make low-level configuration changes. The entries are also
 * mapped into the SYSCTL hierarchy and thus easily accessible.
 */

extern nv_parm_t nv_parms[];

/*
 * These macros extract the encoded ioctl type and number from the
 * command; we inspect the type to verify that device/control ioctls
 * originate from NVIDIA RM clients and use the number to allow the
 * core resource manager's ioctl handler to be ignorant of operating
 * specific ioctl encodings.
 */

#define __NV_IOC_TYPE(_cmd) (((_cmd) >> 8) & 0xff)
#define __NV_IOC_NR(_cmd)   (((_cmd) >> 0) & 0xff)
#define __NV_IOC_SIZE(_cmd) (((_cmd) >> 16) & 0x1fff)

extern uma_zone_t nvidia_stack_t_zone;

#define NV_UMA_ZONE_ALLOC_STACK(ptr)                             \
    {                                                            \
        (ptr) = uma_zalloc(nvidia_stack_t_zone, M_WAITOK);       \
        if ((ptr) != NULL)                                       \
        {                                                        \
            (ptr)->size = sizeof((ptr)->stack);                  \
            (ptr)->top = (ptr)->stack + (ptr)->size;             \
        }                                                        \
    }

#define NV_UMA_ZONE_FREE_STACK(ptr)                              \
    {                                                            \
        uma_zfree(nvidia_stack_t_zone, (ptr));                   \
        (ptr) = NULL;                                            \
    }

/* nvidia_dev.c */
int    nvidia_dev_attach     (struct nvidia_softc *);
int    nvidia_dev_detach     (struct nvidia_softc *);

/* nvidia_ctl.c */
int    nvidia_ctl_attach     (void);
int    nvidia_ctl_detach     (void);

/* nvidia_pci.c */
void   nvidia_pci_check_config_space  (nv_stack_t *, device_t dev, BOOL, BOOL, BOOL);
int    nvidia_pci_setup_intr          (device_t dev);
int    nvidia_pci_teardown_intr       (device_t dev);
U008   nvidia_pci_find_capability     (device_t dev, U008);

#define NV_PCI_CHECK_CONFIG_SPACE(_sp,_nv,_cb,_as,_mb)                   \
    {                                                                    \
        struct nvidia_softc *__sc;                                       \
        if (((_nv)->flags & NV_FLAG_CONTROL) == 0) {                     \
            __sc = (_nv)->os_state;                                      \
            nvidia_pci_check_config_space(_sp,__sc->dev,_cb,_as,_mb);    \
        }                                                                \
    }

/* nvidia_subr.c */
int    nvidia_attach         (device_t);
int    nvidia_detach         (device_t);
int    nvidia_suspend        (device_t);
int    nvidia_resume         (device_t);
int    nvidia_alloc_hardware (device_t);
void   nvidia_free_hardware  (device_t);
void   nvidia_intr           (void *);
int    nvidia_modevent       (module_t, int, void *);

void   nvidia_rc_timer       (void *);

void   nv_lock_api           (nv_state_t *);
void   nv_unlock_api         (nv_state_t *);

S032   nv_alloc_contig_pages (nv_state_t *, U032, U032, NvU64 *, void **);
S032   nv_free_contig_pages  (nv_state_t *, void *);
S032   nv_alloc_system_pages (nv_state_t *, U032, U032, NvU64 *, void **);
S032   nv_free_system_pages  (nv_state_t *, void *);

S032   nv_os_agp_init        (nv_stack_t *sp, nv_state_t *, void **, U032 *);
S032   nv_os_agp_teardown    (nv_stack_t *sp, nv_state_t *);
S032   nv_alloc_agp_pages    (nv_state_t *, U032, U032, void **);
S032   nv_free_agp_pages     (nv_state_t *, U032, void *, U032 *);

/* nvidia_sysctl.c */
void   nvidia_sysctl_init    (void);
void   nvidia_sysctl_exit    (void);

int    nvidia_sysctl_dev_model  (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_dev_vbios  (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_bus_type   (SYSCTL_HANDLER_ARGS);

int    nvidia_sysctl_registry_key     (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_registry_dwords  (SYSCTL_HANDLER_ARGS);

int    nvidia_sysctl_agp_rates      (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_fw         (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_sba        (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_registers  (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_driver     (SYSCTL_HANDLER_ARGS);

int    nvidia_sysctl_agp_status       (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_rate_status  (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_fw_status    (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_sba_status   (SYSCTL_HANDLER_ARGS);

void   nv_sysctl_init        (nv_state_t *);
void   nv_sysctl_exit        (nv_state_t *);

/* nvidia_linux.c */
void   nvidia_linux_init     (void);
void   nvidia_linux_exit     (void);

/* nvidia_os_registry.c */

void   nvidia_update_registry (char *);

/* ioctl helpers */
int    nvidia_get_card_info  (struct nv_ioctl_card_info *);
int    nvidia_handle_ioctl   (struct cdev *, u_long, caddr_t, int, d_thread_t *);

/* device helpers */
int    nvidia_open_ctl       (struct cdev *, d_thread_t *);
int    nvidia_open_dev       (struct nvidia_softc *, struct cdev *, d_thread_t *);
int    nvidia_close_ctl      (struct cdev *, d_thread_t *);
int    nvidia_close_dev      (struct nvidia_softc *, struct cdev *, d_thread_t *);
int    nvidia_mmap_dev       (struct nvidia_softc *, vm_offset_t, vm_offset_t *);

#endif /* __NV_FREEBSD_H__ */
