Adding PTP to io-sock Network Drivers

Summary

This driver design assumes that the hardware uses a free-running internal oscillator. No adjustments are made in the hardware to try to control the internal oscillator's frequency. Adjusting the frequency of an oscillator (e.g., using a Phase-Locked Loop (PLL)) is slow and prone to gain peaking effects. Instead of hardware frequency adjustments, the driver uses the addend value it receives from the PTP module in the sample_ptp_set_compensation() call to change how much time is added to the PTP hardware clock after each tick of the internal oscillator. This solution allows the internal oscillator to free run and its frequency varies over time due to temperature and other variables. However, the PTP daemon adjusts to the changes and keeps the PTP client (slave) synchronized to the PTP server (master) by adjusting the addend passed to the module. This adjustment causes the module to call the ptp_write_comp() function.

PTP module support

Adding PTP support requires ptp_mod.h, which declares functions that are implemented by the PTP module. Networking drivers use these functions to handle actions such as transmit (Tx) and receive (Rx) timestamping and handing off PTP I/O-command (ioctl()) processing. For detailed information about these functions, go to PTP Module Library API.

The ptp_mod.h header file also includes ptp.h, which declares user-space macros and structures like the IEEE 1588 PTP message header and the SIOCSDRVSPEC and SIOCGDRVSPEC ioctl() commands.

Additional FreeBSD driver module setup is required to use the DEVMETHOD() functions. The module calls these functions when it receives I/O-control (ioctl()) commands and ptp_*() function calls. They include functions for setup, Tx/Rx, and PTP module and hardware setup.

#include <qnx/ptp_mod.h>
struct sample_softc {
	/* ... Driver-specific context members ... */
	pthread_mutex_t        hw_rw_mutex;
	if_t                   ifp;
	device_t               dev;
	struct tx_desc         txq[NUM_TXDS];
	struct rx_desc         rxq[NUM_RXDS];
	bool                   is_attached;

	/* PTP-specific data structures */
	struct ptp_ctx     *ptp;
	uint32_t           addend;
	uint32_t           ptp_clk_rate;
	int                ptp_disable;
}


/* PTP module functions */
static int sample_ptp_set_time(device_t dev, struct ptp_time *time);
static int sample_ptp_get_time(device_t dev, struct ptp_time *time);
static int sample_ptp_set_compensation(device_t dev, struct ptp_comp *comp);
static int sample_ptp_get_compensation(device_t dev, struct ptp_comp *comp);
static bool sample_ptp_compare_header(device_t dev, struct ptp_uhdr *mod,
    struct ptp_uhdr *drv);

/* driver-specific setup */
static int sample_ptp_init_hw(struct sample_softc *sc);

/* generic driver setup */
static int sample_ioctl(if_t ifp, u_long cmd, caddr_t data);
static int sample_attach(device_t dev);
static int sample_detach(device_t dev);

/* generic driver Tx/Rx */
static int sample_tx(struct sample_softc *sc);
static int sample_txfinish(struct sample_softc *sc);
static int sample_tstmp_intr(struct sample_softc *sc, struct sample_event *evt);
static int sample_rx(struct sample_softc *sc);

The following prototypes use the syntax that the PTP module expects when these functions are called. They are called by the module and should be implemented by the driver via the DEVMETHOD() functions:

/**
 * @brief Read the current PTP clock time
 *
 * @details Called with @c ptp_mtx locked. @c ptp_time must be within the bounds:
 * - @c ptp_time.sec <= @c MAX_SECONDS (provided in @e ptp_attach())
 * - @c ptp_time.sec > @c 0
 * - @c ptp_time.nsec < @c 1000000000 (equivalent to 1s)
 * - @c ptp_time.nsec > @c 0
 * This function is required.
 *
 * @param[in]  dev      A driver's @c device_t struct given on @e ptp_attach().
 * @param[out] ptp_time Current PTP clock time.
 *
 * @return EOK on success, !EOK otherwise.
 */
int ptp_read_time(device_t dev, struct ptp_time *ptp_time);

/**
 * @brief Set the PTP clock to the provided time
 *
 * @details Called with @c ptp_mtx locked. You may assume @c ptp_time follows
 * the same bounds as required in @e ptp_read_time(). This function is required.
 *
 * @param[in] dev      A driver's @c device_t struct given on @e ptp_attach().
 * @param[in] ptp_time Provided time to write to the PTP clock.
 *
 * @return EOK on success, !EOK otherwise.
 */
int ptp_write_time(device_t dev, struct ptp_time *ptp_time);

/**
 * @brief Write the PTP clock's clock adjustments
 *
 * @details This function is required. Updates the logical clock addend hardware.
 * Syntonization in PTP is optional, but mandatory in gPTP.
 *
 * @param[in] dev  A driver's @c device_t struct given on @e ptp_attach().
 * @param[in] comp Requested addend adjustment value and sign.
 *
 * @return EOK on success, !EOK otherwise.
 */
int ptp_write_comp(device_t dev, struct ptp_comp *comp);

/**
 * @brief Read the PTP clock's clock adjustments
 *
 * @details This function is not required.
 *
 * @param[in]  dev  A driver's @c device_t struct given on @e ptp_attach().
 * @param[out] comp PTP clock's current addend adjustment value and sign.
 *
 * @return EOK on success, !EOK otherwise.
 */
int ptp_read_comp(device_t dev, struct ptp_comp *comp);

/**
 * @brief Compare two @c ptp_uhdr structures.
 *
 * @details This function is required @b if the Tx timestamp is returned via
 * an interrupt, @b AND @c ptp_txintr_finish() is called. It should be
 * constructed with the @c PTP_UHDR_* macros, and must return @c true at the end
 * of the function. @e mod will have its values in host byte order.
 *
 * @param[in] dev A driver's @c device_t struct given on @e ptp_attach().
 * @param[in] mod @c ptp_uhdr copied from @e ptp_txintr().
 * @param[in] drv @c ptp_uhdr provided as an argument to @e ptp_txintr_finish().
 */
int ptp_compare_hdr(device_t dev, struct ptp_uhdr *mod, struct ptp_uhdr *drv);
            

The driver calls the PTP module functions via the DEVMETHOD() associated with the ptp_*() function, as declared in the device_methods_t structure.

static device_method_t sample_methods[] = {
	DEVMETHOD(device_attach, sample_attach),
	DEVMETHOD(device_detach, sample_detach),
	/* ... other device methods ... */

	/* PTP methods */

	/* both ptp_read_time and ptp_write_time are locked by a mutex */
	DEVMETHOD(ptp_read_time, sample_ptp_get_time),
	DEVMETHOD(ptp_write_time, sample_ptp_set_time),
	DEVMETHOD(ptp_write_comp, sample_ptp_set_compensation),

	/*
	 * both ptp_read_comp and ptp_compare_hdr are optional though
	 * ptp_compare_hdr must be defined if you call ptp_txintr_finish()
	 */
	DEVMETHOD(ptp_read_comp, sample_ptp_get_compensation),
	DEVMETHOD(ptp_compare_hdr, sample_ptp_compare_header),
	DEVMETHOD_END
};

static driver_t sample_driver = {
	"sample", sample_methods, sizeof(struct sample_softc)
};

static devclass_t sample_devclass;
DRIVER_MODULE(sample, simplebus, sample_driver, sample_devclass, 0 , 0);
MODULE_DEPEND(sample, ptp, 1, 1, 1);

PTP module functions

Below are examples of PTP module functions as defined in a driver. The ptp_write_comp() function has the largest impact on driver performance.

/* PTP module functions */
static int sample_ptp_get_time(device_t dev, struct ptp_time *time)
{
	/* get the current time from hardware. */

	return (EOK);
}

static int sample_ptp_set_time(device_t dev, struct ptp_time *time)
{
	/* write the new time to hardware */

	return (EOK);
}

static int sample_ptp_set_compensation(device_t dev, struct ptp_comp *comp)
{
	uint32_t new_addend;
	uint32_t delta;
	uint64_t adjustment;
	struct sample_softc *sc = device_get_softc(dev);

	adjustment = ((uint64_t)sc->addend)*((uint64_t)comp->comp);
	delta = (uint32_t)(adjustment / 1000000000ULL);
	if (comp->positive) {
		new_addend = sc->addend + delta;
	} else {
		new_addend = sc->addend - delta;
	}

	/* write new_addend to hardware */

	return (EOK);
}

static int sample_ptp_get_compensation(device_t dev, struct ptp_comp *comp)
{
	/* get the current addend from hardware, unless saved in sc */
	return (EOK);
}

static bool sample_ptp_compare_header(device_t dev, struct ptp_uhdr *mod,
    struct ptp_uhdr *drv)
{
	/*
	 * This function is called when ptp_txintr_finish() is called. The
	 * PTP_UHDR_* macros should be used, with a return of true at the end
	 * of the function. As well, the macros should check the header in
	 * increasing likelihood of equality. i.e., seqid is unlikely to be the
	 * same between multiple packets, in contrast to domain, which may always
	 * be the same. Example:
	 */

	PTP_UHDR_SEQID(mod, drv);
	PTP_UHDR_MSGTYPE(mod, drv);
	PTP_UHDR_DOMAIN(mod, drv);

	return (true);
}

Initialization

The device_t structure is the pointer type for the device structure. All API functions have the device pointer as a first parameter.

When an Ethernet device is found during initialization, the compatible sample_attach() function is called.

From the PTP perspective, the attach function is executed for each Ethernet device detected. The attach function should contain everything needed to initialize the hardware, allocate resources, initialize the MAC and PHY, attach to the interrupts required to transmit and receive packets, and so on.

An attach function attaches a driver to a device. The following example is a simplified example of an Ethernet driver connecting to an Ethernet device and adding in the PTP functionality. The PTP driver functionality is enabled in sample_attach(), and ptp_attach() is usually called after the Ethernet interface has been created. Because ptp_attach() calls ptp_write_time(), all PTP hardware initialization should also be completed before it is called. All the Ethernet initialization is done inside sample_attach() and not before it is called.

static int sample_attach(device_t dev)
{
	struct sample_softc *sc;
	int rc;

	sc = device_get_softc(dev);
	sc->dev = dev;

	/* Initialize hardware access mutex (io-sock is multi-threaded) */
	rc = pthread_mutex_init(&sc->hw_rw_mutex, NULL)
	if (rc != EOK) {
		device_printf(dev, "Failed to init hardware access mutex\n");
		return (rc);
	}

	/* Ethernet interface is created and fully configured */
	if_setioctlfn(sc->ifp, sample_ioctl)
	sc->is_attached = true;

	/* Initialize the PTP hardware */
	rc = sample_ptp_init_hw(sc);
	if (rc != EOK) {
		device_printf(dev, "PTP HW initialization failed\n");
		return (rc);
	}

	/* Initialize the PTP module */
	rc = ptp_attach(dev, &sc->ptp, device_get_nameunit(dev), MAX_SECONDS);
	if (rc != EOK) {
		device_printf(dev, "PTP module initialization failed\n");
		return (rc);
	}
	sc->ptp_disable = false;

	return (rc);
}

static int sample_ptp_init_hw(struct sample_softc *sc)
{
	pthread_mutex_lock(&sc->hw_rw_mutex);

       struct ptp_comp comp;

	/* Initialize the clocking hardware */

	pthread_mutex_unlock(&sc->hw_rw_mutex);

	/*
	 * The addend parameter is the time adjustment used (the increment
	 * added to the clock on every tick) in the PTP clock handling. The
	 * initial value should be calculated based on the PTP clocking hardware
	 * and the clock initialization above.
	 * 
	 * Note: sample_set_addend() is also called frequently on a PTP client
	 * (slave) with adjustment requests from the PTP module. The adjustment
	 * call is made through the PTP module functions.
	 */

	sc->addend = STARTUP_ADDEND;
	comp->comp = sc->addend;
	comp->positive = true;
	sample_ptp_set_compensation(sc, &comp);

	pthread_mutex_lock(&sc->hw_rw_mutex);

	/* Configure the Rx and Tx timestamping hardware */

	pthread_mutex_unlock(&sc->hw_rw_mutex);

	return (EOK);
}

Hardware initialization routines

The two functions sample_ptp_get_time() and sample_ptp_set_time() support the device_method_t interface into the PTP driver from the PTP module. The sample_ptp_get_time() function reads the PTP hardware time in seconds and nanoseconds and returns the value in ptp_time. The sample_ptp_set_time() function updates the PTP hardware time from the module.

In many cases, implementations also use these functions during hardware initialization to verify that the PTP clock is performing as expected. Depending on the platform hardware, there may be other, better methods to verify the timestamping hardware initialization.

Detaching: stopping PTP

For the code details of a driver without PTP, see the sample_detach() function in A Hardware-Independent Sample Driver: sample.c. In most drivers, PTP support is one of the last pieces of functionality turned on in sample_attach(). Therefore, it should be one of the first pieces of functionality turned off in sample_detach().

static int sample_detach(device_t dev)
{
	struct sample_softc *sc;
	int rc;

	/* Disable anything that could call a ptp_* function, then detach */

	sc = device_get_softc(dev);
	sc->ptp_disable = true;
	if (sc->ptp != NULL) {
		ptp_detach(sc->ptp);
	}

	/* Disable interrupts, release hardware and mutexes, etc. */

	pthread_mutex_destroy(&sc->hw_rw_mutex);

	return (EOK);
}   

The ioctl() interface

The PTP module supports the following PTP daemon ioctl() commands, which can be found along with some PTP structures in io-sock/ptp.h. The driver’s ioctl() function must call ptp_ioctl() when processing either a SIOCSDRVSPEC or SIOCGDRVSPEC command, in case the call is a PTP command:

static int sample_ioctl(if_t ifp, u_long cmd, caddr_t data)
{
	struct sample_softc *sc = if_getsoftc(ifp);
	struct ifdrv *ifd = (struct ifdrv *)data;
	int err = EOK;

	switch(cmd) {
	/* all other cases */
	case SIOCSDRVSPEC:
	case SIOCGDRVSPEC:
		if (!sc->ptp_disable) {
			err = ptp_ioctl(sc->ptp, ifd);
		}
		/* other *DRVSPEC handling as needed */
		break;
	}

	return (err);
}

Transmit packet handling

When using ptp_txintr(), the tag is unlinked from the mbuf and queued to wait for the interrupt to return. While queued, if the timestamp has become stale (i.e., exceeded the maximum timeout), it is freed. By default, the timeout is 3 seconds.

When using ptp_txfinish(), the tag is still linked with the mbuf, and no expiration is calculated. In-packet timestamping is preferred because it requires less processing.

The following example illustrates both methods, starting with the transmission, followed by the action of returning the timestamp with both the in-packet and interrupt method.

static int sample_tx(struct sample_softc *sc)
{
	struct mbuf *m;

	/* On transmission w/ the in-packet method */
	if (PTP_TX_TIMESTAMP(m)) {
		/* setup Tx description or other requirements for Tx timestamping */
	}

	/* else, if using the interrupt method */
	if (PTP_TX_TIMESTAMP(m)) {
		/* setup Tx description or other requirements for Tx timestamping */
		ptp_txintr(sc->ctx, m);
	}
}

static int sample_txfinish(struct sample_softc *sc)
{
	struct mbuf *m;
	struct ptp_time ptp_time;
	int i;

	/*
	 * The in-packet Tx timestamping assumes that the packet mbuf has not
	 * been freed, and the timestamp can be tied back to the mbuf,
	 * i.e., Tx descriptor. Since the descriptor should be checked before the
	 * mbuf is freed, we can check if a timestamp has been returned for this
	 * descriptor, and return it
	 */

	for (i = cidx; i < pidx; i++) {
		if (!sc->txqs[i].complete) {
			break;
		}

		/* txd cleanup */
		if (sc->txqs[i].timestamp) {
			ptp_time.sec = timestamp.seconds;
			ptp_time.nsec = timestamp.nanoseconds;
			ptp_txfinish(sc->ptp , m, &ptp_time);
		}
		m_freem(m);
	}
}

static int sample_tstmp_intr(struct sample_softc *sc, struct sample_event *evt)
{
	struct ptp_uhdr uhdr;
	struct ptp_time ptp_time;

	/*
	 * Interrupt-based Tx timestamping requires either a comparison
	 * (ptp_compare_hdr()) or an assumption of transmission -> interrupt
	 * without overlapping Tx timestamps, i.e., FIFO
	 * 
	 * The mbuf may have been freed, but the m_tag is still queued in the
	 * PTP module
	 */

	switch(evt->type) {
	case TX_TSTMP:
		ptp_time.sec = evt->timestamp.sec;
		ptp_time.nsec = evt->timestamp.nsec;

		/* if using hdr comparison */
		/* copying over the information from the interrupt to the uhdr */
		uhdr.members = evt->hdr_info;
		ptp_txintr_finish(sc->ptp, &uhdr, &ptp_time);
		
		/* else, if using FIFO */
		ptp_txintr_finish_head(sc->ptp, &ptp_time);
		break;
	}
}

Receive packet handling

The timestamp is collected before the receive packet is sent to the io-sock stack. The ptp_rx() function fills in the rcv_tstmp field of the mbuf with the time provided. The ptp_rx() function also allocates an m_tag, which is not needed if the mbuf is later freed by an error condition. To save time, call ptp_rx() only after all error conditions that can free the mbuf are checked.

static int sample_rx(struct sample_softc *sc)
{
	struct mbuf *m;
	struct ptp_time ptp_time;
	int i;

	/* we've received a packet */
	for (i = cidx; i < pidx; i++) {
		/* Rx data cleanup and mbuf alloc */
		m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
		/* packet in mbuf now */

		if (sc->rxqs[i].timestamp) {
			ptp_time.sec = timestamp.seconds;
			ptp_time.nsec = timestamp.nanoseconds;
			/*
			 * to save time, ptp_rx() should only be called when
			 * certain no more errors can stop its input
			 */
			ptp_rx(sc->ptp , m, &ptp_time);
		}
	}

	if_input(m);
}