daq.h

#ifndef __PCIE40_DRIVER_DAQ_H
#define __PCIE40_DRIVER_DAQ_H
//p40driver``+

#include "common.h"
#include <linux/fs.h>
#include <linux/mm_types.h>
#include <linux/moduleparam.h>
#include <linux/mutex.h>
#include <linux/poll.h>
#include <linux/sched.h>

#include "pcie40_ioctl.h"

#define P40_DAQ_CDEV_BASEMINOR (2)
#define P40_DAQ_CDEV_COUNT (4)

#define MAIN_MAP_MAX_ENTRIES (1024) // This should be read from the FPGA

//ug`pcie40_driver.options`mainmibs *mainmibs* = _M_::
// Amount of memory (in MiBs) to allocate for each _main_ stream circular buffer. The driver will try to allocate this much memory but the final DMA buffer might be smaller in case of memory pressure from the DMA allocator. The default (and maximum) value is 4096.
#define MAIN_BUF_MIBS_MAX (4096)
static int mainmibs = MAIN_BUF_MIBS_MAX;
module_param(mainmibs, int, S_IRUSR | S_IRGRP | S_IROTH);
MODULE_PARM_DESC(mainmibs, "Desired size (MiB) of main stream circular buffer");

#define META_MAP_MAX_ENTRIES (32) // This should be read from the FPGA

//ug`pcie40_driver.options`metamibs *metamibs* = _M_::
// Amount of memory (in MiBs) to allocate for each _meta_ stream circular buffer. The default (and maximum) value is 128.
#define META_BUF_MIBS_MAX (128) // TODO: this could potentially be reduced to 64
static int metamibs = META_BUF_MIBS_MAX;
module_param(metamibs, int, S_IRUSR | S_IRGRP | S_IROTH);
MODULE_PARM_DESC(metamibs, "Desired size (MiB) of meta stream circular buffer");

//ug`pcie40_driver.files`ctrl _ /dev/pcie40_?_ctrl::
// Device to access the DMA controller.
#define CTRL_CDEV_MINOR (P40_DAQ_CDEV_BASEMINOR + 0)
static const char CTRL_CDEV_NAME[] = "ctrl";

//ug`pcie40_driver.files`main _ /dev/pcie40_?_main::
// Device to access the _main_ DMA stream.
#define MAIN_CDEV_MINOR (P40_DAQ_CDEV_BASEMINOR + 1)
static const char MAIN_CDEV_NAME[] = "main";

//ug`pcie40_driver.files`meta _ /dev/pcie40_?_meta::
// Device to access the _meta_ DMA stream.
#define META_CDEV_MINOR (P40_DAQ_CDEV_BASEMINOR + 2)
static const char META_CDEV_NAME[] = "meta";

////ug`pcie40_driver.files`odin _ /dev/pcie40_?_odin::
//// Device to access the _odin_ DMA stream.
#define ODIN_CDEV_MINOR (P40_DAQ_CDEV_BASEMINOR + 3)
static const char ODIN_CDEV_NAME[] = "odin";

struct pcie40_dma_buffer {
  void *ptr;
  dma_addr_t start;
  size_t size;
};

struct pcie40_dma_map {
  void __iomem* base;
  size_t max_entries;
  size_t num_entries;
  struct pcie40_dma_buffer *entries;
  size_t size;
};

struct pcie40_daq_state;

//+`pcie40_dma_stream` Kernelspace representation of a DMA stream between the PCIe40 FPGA and upstream memory.
struct pcie40_dma_stream {
  const char *cdev_name;
  int8_t cdev_minor;
  struct cdev cdev;
  struct pcie40_dma_map map;
  size_t regs_base;
  uint32_t write_off;
  uint32_t read_off;
  int msi;
  ktime_t msi_last;
  ktime_t msi_delay;
  int msi_count;

  // This lock prevents the read pointer to be changed when we're calculating how much data is available in the circular buffer (the write buffer can be advanced by the FPGA but this does not invalidate the results)
  spinlock_t off_lock;
  unsigned long off_flags;

  struct pcie40_daq_state *state;
};

struct pcie40_daq_state {
  struct pcie40_state *common;

  int msi_base;
  size_t msi_span;
  wait_queue_head_t wait;

  dev_t dev_num; //base MAJOR/MINOR numbers for device files
  struct cdev ctrl_cdev;

  struct pcie40_dma_stream main_stream;
  struct pcie40_dma_stream meta_stream;
  struct pcie40_dma_stream odin_stream;

  u8 irq_line;
  u8 irq_pin;

#ifdef PCIE40_EMU
  struct task_struct *emu_thread;
#endif
};

static struct class *pcie40_daq_class = NULL;

#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35))
static int ctrl_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
#else
static long ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
#endif //+`ctrl_ioctl`
{
  struct pcie40_daq_state *state = filp->private_data;
  uint64_t __user *arg_ptr = (uint64_t __user *)arg;
  int err = 0;
  uint64_t value;

  if (_IOC_DIR(cmd) & _IOC_READ)
    err = !access_ok(VERIFY_WRITE, arg_ptr, _IOC_SIZE(cmd));
  else if (_IOC_DIR(cmd) & _IOC_WRITE)
    err = !access_ok(VERIFY_READ, arg_ptr, _IOC_SIZE(cmd));
  if (err) return -EFAULT;

  switch(cmd) {
    case P40_CTRL_GET_RWTEST: //ioctl.pcie`P40_CTRL_GET_RWTEST`
      // See regmap`pcie.dma_ctrl.rwtest` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_RWTEST);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_RWTEST: //ioctl.pcie`P40_CTRL_SET_RWTEST`
      // See regmap`pcie.dma_ctrl.rwtest` .
      __get_user(value, arg_ptr);
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_RWTEST, value);
      break;
    case P40_CTRL_GET_VERSION: //ioctl.pcie`P40_CTRL_GET_VERSION`
      // See regmap`pcie.dma_ctrl.version` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_VERSION);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_LINK_ID: //ioctl.pcie`P40_CTRL_GET_LINK_ID`
      // The value returned by this IOCTL encodes both the PCI topological address and the FPGA interface number, as follows:
      // _ bits [31..24]::
      // PCI bus number
      // _ bits [23..16]::
      // PCI slot number
      // _ bits [15..8]::
      // PCI function number
      // _ bits [7..0]::
      // FPGA-PCIe interface number within a given board (0 for the primary link and 1 for the secondary)
      value = state->common->link_id;
      if (state->common->pci_dev) {
        value |= state->common->pci_dev->bus->number << 24;
        value |= PCI_SLOT(state->common->pci_dev->devfn) << 16;
        value |= PCI_FUNC(state->common->pci_dev->devfn) << 8;
      }
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_CHIP_ID: //ioctl.pcie`P40_CTRL_GET_CHIP_ID`
      // See regmap`pcie.dma_ctrl.chip_id` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHIP_ID_HI);
      value <<= 32;
      value |= pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHIP_ID_LO);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_RESET: //ioctl.pcie`P40_CTRL_GET_RESET`
      // See regmap`pcie.dma_ctrl.reset` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_RESET);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_RESET: //ioctl.pcie`P40_CTRL_SET_RESET`
      // See regmap`pcie.dma_ctrl.reset` .
      __get_user(value, arg_ptr);
      if (value) {
        state->main_stream.msi_count = 0;
        state->meta_stream.msi_count = 0;
      }
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_RESET, value);
      break;
    case P40_CTRL_GET_ERROR: //ioctl.pcie`P40_CTRL_GET_ERROR`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_ERROR);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_MAIN_GEN_CTL: //ioctl.pcie`P40_CTRL_GET_MAIN_GEN_CTL`
      // See regmap`pcie.dma_ctrl.main_gen_ctl` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_MAIN_GEN_CTL);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_MAIN_GEN_CTL: //ioctl.pcie`P40_CTRL_SET_MAIN_GEN_CTL`
      // See regmap`pcie.dma_ctrl.main_gen_ctl` .
      __get_user(value, arg_ptr);
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_MAIN_GEN_CTL, value);
      break;
    case P40_CTRL_GET_MAIN_GEN_FIXED: //ioctl.pcie`P40_CTRL_GET_MAIN_GEN_FIXED`
      // See regmap`pcie.dma_ctrl.main_gen_fixed` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_MAIN_GEN_FIXED);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_MAIN_GEN_FIXED: //ioctl.pcie`P40_CTRL_SET_MAIN_GEN_FIXED`
      // See regmap`pcie.dma_ctrl.main_gen_fixed` .
      __get_user(value, arg_ptr);
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_MAIN_GEN_FIXED, value);
      break;
    case P40_CTRL_GET_MAIN_RAW_MODE: //ioctl.pcie`P40_CTRL_GET_MAIN_RAW_MODE`
      // See regmap`pcie.dma_ctrl.main_raw_mode` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_MAIN_RAW_MODE);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_META_PACKING: //ioctl.pcie`P40_CTRL_GET_META_PACKING`
      // See regmap`pcie.dma_ctrl.meta_packing` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_META_PACKING);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_META_PACKING: //ioctl.pcie`P40_CTRL_SET_META_PACKING`
      __get_user(value, arg_ptr);
      // See regmap`pcie.dma_ctrl.meta_packing` .
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_META_PACKING, value);
      break;
    case P40_CTRL_GET_PCIE_GEN: //ioctl.pcie`P40_CTRL_GET_PCIE_GEN`
      // See regmap`pcie.dma_ctrl.pcie_gen` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_PCIE_GEN);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_MSI_MODE: //ioctl.pcie`P40_CTRL_GET_MSI_MODE`
      // See regmap`pcie.dma_ctrl.msi_mode` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_MSI_MODE);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_MSI_MODE: //ioctl.pcie`P40_CTRL_SET_MSI_MODE`
      // See regmap`pcie.dma_ctrl.msi_mode` .
      __get_user(value, arg_ptr);
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_MSI_MODE, value);
      break;
    case P40_CTRL_GET_MAIN_MSI_BYTES: //ioctl.pcie`P40_CTRL_GET_MAIN_MSI_BYTES`
      // See regmap`pcie.dma_ctrl.main_msi_bytes` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_MAIN_MSI_BYTES);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_MAIN_MSI_BYTES: //ioctl.pcie`P40_CTRL_SET_MAIN_MSI_BYTES`
      __get_user(value, arg_ptr);
      // See regmap`pcie.dma_ctrl.main_msi_bytes` .
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_MAIN_MSI_BYTES, value);
      break;
    case P40_CTRL_GET_MAIN_MSI_CYCLES: //ioctl.pcie`P40_CTRL_GET_MAIN_MSI_CYCLES`
      // See regmap`pcie.dma_ctrl.main_msi_cycles` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_MAIN_MSI_CYCLES);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_META_MSI_BYTES: //ioctl.pcie`P40_CTRL_GET_META_MSI_BYTES`
      // See regmap`pcie.dma_ctrl.meta_msi_bytes` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_META_MSI_BYTES);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_META_MSI_BYTES: //ioctl.pcie`P40_CTRL_SET_META_MSI_BYTES`
      // See regmap`pcie.dma_ctrl.meta_msi_bytes` .
      __get_user(value, arg_ptr);
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_META_MSI_BYTES, value);
      break;
    case P40_CTRL_GET_META_MSI_CYCLES: //ioctl.pcie`P40_CTRL_GET_META_MSI_CYCLES`
      // See regmap`pcie.dma_ctrl.meta_msi_cycles` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_META_MSI_CYCLES);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_META_MSI_BLOCKS: //ioctl.pcie`P40_CTRL_GET_META_MSI_BLOCKS`
      // See regmap`pcie.dma_ctrl.meta_msi_blocks` .
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_META_MSI_BLOCKS);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_META_MSI_BLOCKS: //ioctl.pcie`P40_CTRL_SET_META_MSI_BLOCKS`
      // See regmap`pcie.dma_ctrl.meta_msi_blocks` .
      __get_user(value, arg_ptr);
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_META_MSI_BLOCKS, value);
      break;
    case P40_CTRL_GET_HOST_MAIN_MSI_NSECS: //ioctl.pcie`P40_CTRL_GET_HOST_MAIN_MSI_NSECS`
      value = ktime_to_ns(state->main_stream.msi_delay);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_HOST_META_MSI_NSECS: //ioctl.pcie`P40_CTRL_GET_HOST_META_MSI_NSECS`
      value = ktime_to_ns(state->meta_stream.msi_delay);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_INBUF_SIZE: //ioctl.pcie`P40_CTRL_GET_INBUF_SIZE`
      // See regmap`pcie.dma_ctrl.inbuf_size`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_INBUF_SIZE);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_INBUF_FILL: //ioctl.pcie`P40_CTRL_GET_INBUF_FILL`
      // See regmap`pcie.dma_ctrl.inbuf_fill`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_INBUF_FILL);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_TRUNC_THRES: //ioctl.pcie`P40_CTRL_GET_TRUNC_THRES`
      // See regmap`pcie.dma_ctrl.trunc_thres`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_THRES);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_SET_TRUNC_THRES: //ioctl.pcie`P40_CTRL_SET_TRUNC_THRES`
      // See regmap`pcie.dma_ctrl.trunc_thres`
      __get_user(value, arg_ptr);
      pcie40_write32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_THRES, value);
      break;
    case P40_CTRL_GET_TRUNC_TOTAL_CYCLES: //ioctl.pcie`P40_CTRL_GET_TRUNC_TOTAL_CYCLES`
      // See regmap`pcie.dma_ctrl.trunc_total_cycles`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_TOTAL_CYCLES);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_TRUNC_TOTAL_SINCE_EVID: //ioctl.pcie`P40_CTRL_GET_TRUNC_TOTAL_SINCE_EVID`
      // See regmap`pcie.dma_ctrl.trunc_total_since_evid`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_TOTAL_SINCE_EVID_HI);
      value <<= 32;
      value |= pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_TOTAL_SINCE_EVID_LO);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_TRUNC_LAST_CYCLES: //ioctl.pcie`P40_CTRL_GET_TRUNC_LAST_CYCLES`
      // See regmap`pcie.dma_ctrl.trunc_last_cycles`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_LAST_CYCLES);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_TRUNC_LAST_FROM_EVID: //ioctl.pcie`P40_CTRL_GET_TRUNC_LAST_FROM_EVID`
      // See regmap`pcie.dma_ctrl.trunc_last_from_evid`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_LAST_FROM_EVID_HI);
      value <<= 32;
      value |= pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_LAST_FROM_EVID_LO);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_TRUNC_LAST_TO_EVID: //ioctl.pcie`P40_CTRL_GET_TRUNC_LAST_TO_EVID`
      // See regmap`pcie.dma_ctrl.trunc_last_to_evid`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_LAST_TO_EVID_HI);
      value <<= 32;
      value |= pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_TRUNC_LAST_TO_EVID_LO);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_CHOKE_TOTAL_CYCLES: //ioctl.pcie`P40_CTRL_GET_CHOKE_TOTAL_CYCLES`
      // See regmap`pcie.dma_ctrl.choke_total_cycles`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHOKE_TOTAL_CYCLES);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_CHOKE_TOTAL_SINCE_EVID: //ioctl.pcie`P40_CTRL_GET_CHOKE_TOTAL_SINCE_EVID`
      // See regmap`pcie.dma_ctrl.choke_total_since_evid`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHOKE_TOTAL_SINCE_EVID_HI);
      value <<= 32;
      value |= pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHOKE_TOTAL_SINCE_EVID_LO);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_CHOKE_LAST_CYCLES: //ioctl.pcie`P40_CTRL_GET_CHOKE_LAST_CYCLES`
      // See regmap`pcie.dma_ctrl.choke_last_cycles`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHOKE_LAST_CYCLES);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_CHOKE_LAST_FROM_EVID: //ioctl.pcie`P40_CTRL_GET_CHOKE_LAST_FROM_EVID`
      // See regmap`pcie.dma_ctrl.choke_last_from_evid`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHOKE_LAST_FROM_EVID_HI);
      value <<= 32;
      value |= pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHOKE_LAST_FROM_EVID_LO);
      __put_user(value, arg_ptr);
      break;
    case P40_CTRL_GET_CHOKE_LAST_TO_EVID: //ioctl.pcie`P40_CTRL_GET_CHOKE_LAST_TO_EVID`
      // See regmap`pcie.dma_ctrl.choke_last_to_evid`
      value = pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHOKE_LAST_TO_EVID_HI);
      value <<= 32;
      value |= pcie40_read32_ctrl(state->common, P40_DMA_CTRL_OFF_CHOKE_LAST_TO_EVID_LO);
      __put_user(value, arg_ptr);
      break;

/*  unfortunately this is only possible in linux 4.x
    case P40_CTL_GET_GEN:
      switch (state->common->pci_dev->bus->cur_bus_speed) {
        case PCIE_SPEED_2_5GT:
          value = 1;
          break;
        case PCIE_SPEED_5_0GT:
          value = 2;
          break;
        case PCIE_SPEED_8_0GT:
          value = 3;
          break;
        default:
          value = 0;
          break;
      }
      __put_user(value, arg_ptr);
      break;
*/
    default:
      printk(P40_ERR "invalid ioctl\n", P40_PARM);
      return -EINVAL;
  }
  return 0;
}

static unsigned int ctrl_poll(struct file *file, poll_table *wait)
{
  // Make sure device is in MSI_DAQ mode
  // (Do we really want to do this at every poll()? Well it's just a few ns,
  // and it's still better than doing it in the isr)
  printk(P40_DIAG, P40_PARM);

  return 0;
}

//+`ctrl_open`
static int ctrl_open(struct inode *inode, struct file *filp)//;?>
{
  struct pcie40_daq_state *state = container_of(inode->i_cdev, struct pcie40_daq_state, ctrl_cdev);
  //printk(P40_INFO, P40_PARM);

  filp->private_data = state;

  return pcie40_device_accessible(state->common) ? 0 : -EIO;
}

//+`ctrl_release`
static int ctrl_release(struct inode *inode, struct file *filp)//;?>
{
  struct pcie40_daq_state *state = container_of(inode->i_cdev, struct pcie40_daq_state, ctrl_cdev);
  //printk(P40_INFO, P40_PARM);

  if (filp->private_data != state) {
    printk(P40_DIAG "inconsistent private_data\n", P40_PARM);
    return -EINVAL;
  }
  filp->private_data = NULL;

  return 0;
}

static struct file_operations ctrl_file_ops = {
  .owner = THIS_MODULE,
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35))
  .ioctl = ctrl_ioctl,
#else
  .unlocked_ioctl = ctrl_ioctl,
#endif
  .open    = ctrl_open,
  .release = ctrl_release,
  .poll    = ctrl_poll,
};

static int dma_map_mmap(struct pcie40_dma_map* map, struct vm_area_struct *vma);
static int dma_map_munmap(struct pcie40_dma_map* map);
static void dma_map_print(struct pcie40_dma_map* map);

static uint32_t dma_stream_get_read_off(struct pcie40_dma_stream *stream);
static void dma_stream_set_read_off(struct pcie40_dma_stream *stream, uint32_t read_off);
static uint32_t dma_stream_get_write_off(struct pcie40_dma_stream *stream);

static inline size_t dma_stream_get_bytes_used(struct pcie40_dma_stream *stream)
{
  uint64_t host_buf_bytes = stream->map.size;
  uint64_t host_write_off, host_read_off;

  spin_lock_irqsave(&stream->off_lock, stream->off_flags);
  host_write_off = dma_stream_get_write_off(stream); //TODO: use cached value
  host_read_off = dma_stream_get_read_off(stream); //TODO: use cached value
  spin_unlock_irqrestore(&stream->off_lock, stream->off_flags);

  if (host_write_off >= host_read_off) {
    return host_write_off - host_read_off;
  } else {
    return host_buf_bytes - (host_read_off - host_write_off);
  }
}

static inline size_t dma_stream_get_bytes_free(struct pcie40_dma_stream *stream)
{
  return stream->map.size - dma_stream_get_bytes_used(stream);
}

static inline uint32_t pcie40_read32_stream(struct pcie40_dma_stream *stream, unsigned long offset);
static inline void pcie40_write32_stream(struct pcie40_dma_stream *stream, unsigned long offset, uint32_t value);

#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35))
static int dma_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
#else
static long dma_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
#endif
//+`dma_ioctl`
{
  struct pcie40_dma_stream *stream = filp->private_data;
  uint64_t __user *arg_ptr = (uint64_t __user *)arg;
  int err = 0;
  uint64_t value;

  if (_IOC_DIR(cmd) & _IOC_READ)
    err = !access_ok(VERIFY_WRITE, arg_ptr, _IOC_SIZE(cmd));
  else if (_IOC_DIR(cmd) & _IOC_WRITE)
    err = !access_ok(VERIFY_READ, arg_ptr, _IOC_SIZE(cmd));
  if (err) return -EFAULT;

  switch(cmd) {
    case P40_STREAM_GET_ENABLE: //ioctl.pcie`P40_STREAM_GET_ENABLE`
      // See regmap`pcie.dma_daq_stream.enable`
      value = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_ENABLE);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_SET_ENABLE: //ioctl.pcie`P40_STREAM_SET_ENABLE`
      // See regmap`pcie.dma_daq_stream.enable`
      __get_user(value, arg_ptr);
      pcie40_write32_stream(stream, P40_DMA_DAQ_STREAM_OFF_ENABLE, value);
      break;
    case P40_STREAM_GET_READY: //ioctl.pcie`P40_STREAM_GET_READY`
      // See regmap`pcie.dma_daq_stream.ready`
      value = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_READY);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_FLUSH: //ioctl.pcie`P40_STREAM_GET_FLUSH`
      // See regmap`pcie.dma_daq_stream.flush`
      value = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FLUSH);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_SET_FLUSH: //ioctl.pcie`P40_STREAM_SET_FLUSH`
      // See regmap`pcie.dma_daq_stream.flush`
      __get_user(value, arg_ptr);
      pcie40_write32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FLUSH, value);
      break;
    case P40_STREAM_GET_FPGA_BUF_BYTES: //ioctl.pcie`P40_STREAM_GET_FPGA_BUF_BYTES`
      // See regmap`pcie.dma_daq_stream.fpga_buf_bytes`
      value = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FPGA_BUF_BYTES);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_FPGA_BUF_DESCS: //ioctl.pcie`P40_STREAM_GET_FPGA_BUF_DESCS`
      // See regmap`pcie.dma_daq_stream.fpga_buf_descs`
      value = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FPGA_BUF_DESCS);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_FPGA_BUF_DESC_BYTES: //ioctl.pcie`P40_STREAM_GET_FPGA_BUF_DESC_BYTES`
      // See regmap`pcie.dma_daq_stream.fpga_buf_desc_bytes`
      value = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FPGA_BUF_DESC_BYTES);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_FPGA_BUF_DESCS_FILL: //ioctl.pcie`P40_STREAM_GET_FPGA_BUF_DESCS_FILL`
      // See regmap`pcie.dma_daq_stream.fpga_buf_descs_fill`
      value = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FPGA_BUF_DESCS_FILL_HI);
      value <<= 32;
      value |= pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FPGA_BUF_DESCS_FILL_LO);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_FPGA_BUF_DESC_FILL_BYTES: //ioctl.pcie`P40_STREAM_GET_FPGA_BUF_DESC_FILL_BYTES`
      // See regmap`pcie.dma_daq_stream.fpga_buf_desc_fill_bytes`
      value = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FPGA_BUF_DESC_FILL_BYTES);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_FPGA_BUF_DESCS_BUSY: //ioctl.pcie`P40_STREAM_GET_FPGA_BUF_DESCS_BUSY`
      // See regmap`pcie.dma_daq_stream.fpga_buf_descs_busy`
      value = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FPGA_BUF_DESCS_BUSY_HI);
      value <<= 32;
      value |= pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_FPGA_BUF_DESCS_BUSY_LO);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_HOST_BUF_WRITE_OFF: //ioctl.pcie`P40_STREAM_GET_HOST_BUF_WRITE_OFF`
      value = dma_stream_get_write_off(stream);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_HOST_BUF_READ_OFF: //ioctl.pcie`P40_STREAM_GET_HOST_BUF_READ_OFF`
      value = dma_stream_get_read_off(stream);
      __put_user(value, arg_ptr);
      break;
  /*
    case P40_STREAM_SET_HOST_BUF_READ_OFF:
      __get_user(value, arg_ptr);
      iowrite32(value & 0xFFFFFFFF, stream->state->common->bar1_regs
        + stream->regs_base + P40_DMA_DAQ_STREAM_OFF_HOST_BUF_READ_OFF);
      break;
  */
  /*case P40_STREAM_GET_HOST_MAP_ENTRIES:
      value = ioread32(stream->state->common->bar1_regs
        + stream->regs_base + P40_DMA_DAQ_STREAM_OFF_HOST_MAP_ENTRIES);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_SET_HOST_MAP_ENTRIES:
      __get_user(value, arg_ptr);
      iowrite32(value & 0xFFFFFFFF, stream->state->common->bar1_regs
        + stream->regs_base + P40_DMA_DAQ_STREAM_OFF_HOST_MAP_ENTRIES);
      break;*/
    case P40_STREAM_GET_HOST_BUF_BYTES: //ioctl.pcie`P40_STREAM_GET_HOST_BUF_BYTES`
      value = stream->map.size;
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_HOST_BUF_BYTES_USED: //ioctl.pcie`P40_STREAM_GET_HOST_BUF_BYTES_USED`
      value = dma_stream_get_bytes_used(stream);
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_GET_HOST_MSI_COUNT: //ioctl.pcie`P40_STREAM_GET_HOST_MSI_COUNT`
      value = stream->msi_count;
      __put_user(value, arg_ptr);
      break;
    case P40_STREAM_FREE_HOST_BUF_BYTES: //ioctl.pcie`P40_STREAM_FREE_HOST_BUF_BYTES`
    {
      uint64_t host_buf_bytes = stream->map.size;
      uint64_t host_write_off, host_read_off;
      __get_user(value, arg_ptr);

      spin_lock_irqsave(&stream->off_lock, stream->off_flags);
      host_write_off = dma_stream_get_write_off(stream); //TODO: use cached value
      host_read_off = dma_stream_get_read_off(stream); //TODO: use cached value

      if (host_write_off >= host_read_off) {
        value = min(value, host_write_off - host_read_off);
      } else {
        value = min(value, host_buf_bytes - (host_read_off - host_write_off));
      }
      dma_stream_set_read_off(stream, (host_read_off + value) % host_buf_bytes);
      spin_unlock_irqrestore(&stream->off_lock, stream->off_flags);

      __put_user(value, arg_ptr);
      break;
    }
    default:
      printk(P40_ERR "invalid ioctl\n", P40_PARM);
      return -EINVAL;
  }
  return 0;
}

//+`dma_mmap`
static int dma_mmap(struct file* filp, struct vm_area_struct* vma)//;?>
{
  struct pcie40_dma_stream *stream = filp->private_data;
  int ret;

  dma_map_print(&stream->map);

  ret = dma_map_mmap(&stream->map, vma);

  vma->vm_private_data = stream;
  return ret;
}

//+`dma_poll`
static unsigned int dma_poll(struct file *filp, poll_table *wait)//;?>
{
  struct pcie40_dma_stream *stream = filp->private_data;
  uint32_t host_write_off, host_read_off;

  spin_lock_irqsave(&stream->off_lock, stream->off_flags);
  host_write_off = dma_stream_get_write_off(stream);
  host_read_off = stream->read_off;
  spin_unlock_irqrestore(&stream->off_lock, stream->off_flags);

  //printk(P40_INFO, P40_PARM);

  poll_wait(filp, &stream->state->wait, wait);

  if (host_write_off != host_read_off)
    return POLLIN | POLLRDNORM;

  //if (dma-stopped)
  //  return POLLIN | POLLRDNORM; //POLLHUP;

  if (!pcie40_device_accessible(stream->state->common))
    return POLLERR;

  return 0;
}

//+`dma_open`
static int dma_open(struct inode *inode, struct file *filp)//;?>
{
  struct pcie40_dma_stream *stream = container_of(inode->i_cdev, struct pcie40_dma_stream, cdev);
  //printk(P40_INFO, P40_PARM);

  filp->private_data = stream;

  return 0;
}

//+`dma_release`
static int dma_release(struct inode *inode, struct file *filp)//;?>
{
  struct pcie40_dma_stream *stream = container_of(inode->i_cdev, struct pcie40_dma_stream, cdev);
  //printk(P40_INFO, P40_PARM);

  if (filp->private_data != stream) {
    printk(P40_DIAG "inconsistent private_data\n", P40_PARM);
    return -EINVAL;
  }
  filp->private_data = NULL;

  return 0;
}

static struct file_operations dma_file_ops = {
  .owner = THIS_MODULE,
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35))
  .ioctl = dma_ioctl,
#else
  .unlocked_ioctl = dma_ioctl,
#endif
  .mmap    = dma_mmap,
  .poll    = dma_poll,
  .open    = dma_open,
  .release = dma_release,
};

static inline uint32_t pcie40_read32_stream(struct pcie40_dma_stream *stream, unsigned long offset)
{
#ifndef PCIE40_EMU
  return ioread32(stream->state->common->bar1_regs + stream->regs_base + offset);
#else
  return *(uint32_t *)(stream->state->common->bar1_regs + stream->regs_base + offset);
#endif
}

static inline void pcie40_write32_stream(struct pcie40_dma_stream *stream, unsigned long offset, uint32_t value)
{
#ifndef PCIE40_EMU
  iowrite32(value, stream->state->common->bar1_regs + stream->regs_base + offset);
#else
  *(uint32_t *)(stream->state->common->bar1_regs + stream->regs_base + offset) = value;
#endif
}

//+`dma_map_mmap`
static int dma_map_mmap(struct pcie40_dma_map* map, struct vm_area_struct *vma)//?>
{
  int rc;
  int i = 0;
  size_t mapped_bytes = 0;
  unsigned long mapped_end = vma->vm_start;

  printk(P40_INFO "VIRT=%lx PHYS=0x%pad SIZE=%lu\n", P40_PARM,
   vma->vm_start, &(map->entries[0].start), vma->vm_end - vma->vm_start);

  //TODO: make this mapping read only

  if (map->num_entries <= map->max_entries) {
    while (mapped_end < vma->vm_end) {
      struct pcie40_dma_buffer *buffer = map->entries + i;

      rc = remap_pfn_range(vma, mapped_end, buffer->start >> PAGE_SHIFT,
       buffer->size, vma->vm_page_prot);

      mapped_bytes += buffer->size;
      mapped_end += buffer->size;
      if (rc) {
        printk(P40_DIAG "remap_pfn_range() failed\n", P40_PARM);
        //TODO: undo all previous mappings before returning
        return rc;
      }
      i = (i+1) % map->num_entries;
    }
  }
  printk(P40_INFO "mapped %zu bytes\n", P40_PARM, mapped_bytes);
  return 0;
}

static int dma_map_munmap(struct pcie40_dma_map* map)
{
  return 0;
}

static void dma_map_print(struct pcie40_dma_map* map)
{
  printk(P40_DIAG "memory map: %zu entries\n", P40_PARM, map->num_entries);/*
  int i;
  for (i = 0; i < map->num_entries; ++i) {
    struct pcie40_dma_buffer* buffer = map->entries + i;

    printk(P40_DIAG " [%3d] base: 0x%pad size: %zu bytes\n", P40_PARM, i,
     (void *)dma_map_read_entry_base(map, i),
     dma_map_read_entry_size(map, i)
    );
  }*/
}

static uint32_t dma_stream_get_read_off(struct pcie40_dma_stream *stream)
{
  // See regmap`pcie.dma_daq_stream.host_buf_read_off
  //TODO: spin lock to serialize access!
  return (stream->read_off = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_HOST_BUF_READ_OFF));
}

static void dma_stream_set_read_off(struct pcie40_dma_stream *stream, uint32_t read_off)
{
  // See regmap`pcie.dma_daq_stream.host_buf_read_off`
  //TODO: spin lock to serialize access!
  stream->read_off = read_off;
  pcie40_write32_stream(stream, P40_DMA_DAQ_STREAM_OFF_HOST_BUF_READ_OFF, read_off);
}

static uint32_t dma_stream_get_write_off(struct pcie40_dma_stream *stream)
{
  // See regmap`pcie.dma_daq_stream.host_buf_write_off`
  //TODO: spin lock to serialize access!
  return (stream->write_off = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_HOST_BUF_WRITE_OFF));
}

//+`dma_stream_configure`
static int dma_stream_configure(int dev_id, struct pcie40_dma_stream *stream, size_t map_base, size_t map_max_entries, size_t buf_desired_bytes)//;?>
{
  int rc;
  uint32_t enable;

  stream->msi_count = 0;
  spin_lock_init(&stream->off_lock);

  enable = pcie40_read32_stream(stream, P40_DMA_DAQ_STREAM_OFF_ENABLE);
  pcie40_write32_stream(stream, P40_DMA_DAQ_STREAM_OFF_ENABLE, 0);

  rc = pcie40_setup_cdev(pcie40_daq_class, &stream->cdev, stream->state->dev_num, stream->cdev_minor, 1, stream->cdev_name, dev_id, &dma_file_ops);
  if (rc < 0) {
    goto err_setup_cdev;
  }

  printk(P40_INFO "allocating memory (%lu bytes)", P40_PARM, buf_desired_bytes);

#ifndef PCIE40_EMU
  if (dma_map_alloc(stream->state->common->pci_dev, &stream->map, stream->state->common->bar1_regs + map_base, map_max_entries, buf_desired_bytes) < 0) {
    goto err_map_alloc;
  }
#else
  if (dma_map_emu_alloc(&stream->map, stream->state->common->bar1_regs + map_base, map_max_entries, buf_desired_bytes) < 0) {
    goto err_map_alloc;
  }
#endif
  pcie40_write32_stream(stream, P40_DMA_DAQ_STREAM_OFF_HOST_MAP_ENTRIES, stream->map.num_entries);

  pcie40_write32_stream(stream, P40_DMA_DAQ_STREAM_OFF_ENABLE, enable);

  dma_map_print(&stream->map);
  return 0;

err_setup_cdev:

err_map_alloc:
  printk(P40_INFO "remove /dev/pcie40_%d_%s\n", P40_PARM, dev_id, stream->cdev_name);
  device_destroy(pcie40_daq_class, MKDEV(MAJOR(stream->state->dev_num), MINOR(stream->state->dev_num)+stream->cdev_minor));

  return -1;
}

//+`dma_stream_destroy`
static void dma_stream_destroy(int dev_id, struct pcie40_dma_stream *stream)//;?>
{
  printk(P40_INFO "remove /dev/pcie40_%d_%s\n", P40_PARM, dev_id, stream->cdev_name);
  device_destroy(pcie40_daq_class,
   MKDEV(MAJOR(stream->state->dev_num),
   MINOR(stream->state->dev_num)+stream->cdev_minor));

  printk(P40_INFO "free DMA buffer(s)\n", P40_PARM);

#ifndef PCIE40_EMU
  dma_map_free(stream->state->common->pci_dev, &stream->map);
#else
  dma_map_emu_free(&stream->map);
#endif
  pcie40_write32_stream(stream, P40_DMA_DAQ_STREAM_OFF_HOST_MAP_ENTRIES, 0);
}

//+`pcie40_daq_init`
int pcie40_daq_init(void)//;?>
{
  int rc = 0;

  pcie40_daq_class = class_create(THIS_MODULE, PCIE40_DAQ_CLASS);
  if (IS_ERR(pcie40_daq_class)) {
    rc = PTR_ERR(pcie40_daq_class);
    printk(P40_WARN "failed to register class, %d\n", P40_PARM, rc);
    goto err_class_create;
  }
  //pcie40_daq_class->dev_uevent = pcie40_dev_uevent;
  pcie40_daq_class->devnode = pcie40_devnode;

err_class_create:
  return rc;
}

//+`pcie40_daq_exit`
void pcie40_daq_exit(void)//;?>
{
  class_destroy(pcie40_daq_class);
}

#endif//__PCIE40_DRIVER_DAQ_H