lwis_fence.c - kernel/google-modules/lwis - Git at Google

 /*
  * Google LWIS Fence
  *
  * Copyright (c) 2022 Google, LLC
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */

 #include <linux/anon_inodes.h>
 #include <linux/file.h>
 #include <linux/mm.h>
 #include <linux/poll.h>

 #include "lwis_device_top.h"
 #include "lwis_commands.h"
 #include "lwis_fence.h"
 #include "lwis_transaction.h"

 #define HASH_CLIENT(x) hash_ptr(x, LWIS_CLIENTS_HASH_BITS)

 bool lwis_fence_debug;
 module_param(lwis_fence_debug, bool, 0644);

 static int lwis_fence_release(struct inode *node, struct file *fp);
 static ssize_t lwis_fence_get_status(struct file *fp, char __user *user_buffer, size_t len,
 				     loff_t *offset);
 static ssize_t lwis_fence_write_status(struct file *fp, const char __user *user_buffer, size_t len,
 				       loff_t *offset);
 static unsigned int lwis_fence_poll(struct file *fp, poll_table *wait);

 static const struct file_operations fence_file_ops = {
 	.owner = THIS_MODULE,
 	.release = lwis_fence_release,
 	.read = lwis_fence_get_status,
 	.write = lwis_fence_write_status,
 	.poll = lwis_fence_poll,
 };

 static int get_fence_status(struct lwis_fence *lwis_fence)
 {
 	int status;
 	unsigned long flags;
 	spin_lock_irqsave(&lwis_fence->lock, flags);
 	status = lwis_fence->status;
 	spin_unlock_irqrestore(&lwis_fence->lock, flags);
 	return status;
 }

 /*
  *  lwis_fence_release: Closing an instance of a LWIS fence
  */
 static int lwis_fence_release(struct inode *node, struct file *fp)
 {
 	struct lwis_fence *lwis_fence = fp->private_data;
 	struct lwis_fence_trigger_transaction_list *tx_list;
 	struct lwis_pending_transaction_id *transaction_id;
 	/* Temporary vars for traversal */
 	struct hlist_node *n;
 	struct list_head *it_tran, *it_tran_tmp;
 	int i;

 	if (lwis_fence_debug) {
 		dev_info(lwis_fence->lwis_top_dev->dev, "Releasing lwis_fence fd-%d",
 			 lwis_fence->fd);
 	}

 	if (lwis_fence->status == LWIS_FENCE_STATUS_NOT_SIGNALED) {
 		dev_err(lwis_fence->lwis_top_dev->dev,
 			"lwis_fence fd-%d release without being signaled", lwis_fence->fd);
 	}

 	if (!hash_empty(lwis_fence->transaction_list)) {
 		hash_for_each_safe (lwis_fence->transaction_list, i, n, tx_list, node) {
 			if (!list_empty(&tx_list->list)) {
 				list_for_each_safe (it_tran, it_tran_tmp, &tx_list->list) {
 					transaction_id =
 						list_entry(it_tran,
 							   struct lwis_pending_transaction_id,
 							   list_node);
 					list_del(&transaction_id->list_node);
 					kfree(transaction_id);
 				}
 			}
 			hash_del(&tx_list->node);
 			kfree(tx_list);
 		}
 	}

 	kfree(lwis_fence);
 	return 0;
 }

 /*
  *  lwis_fence_get_status: Read the LWIS fence's status
  */
 static ssize_t lwis_fence_get_status(struct file *fp, char __user *user_buffer, size_t len,
 				     loff_t *offset)
 {
 	int status = 0;
 	struct lwis_fence *lwis_fence = fp->private_data;
 	int max_len, read_len;

 	if (!lwis_fence) {
 		return -EFAULT;
 	}

 	max_len = sizeof(status) - *offset;
 	if (len > max_len) {
 		len = max_len;
 	}

 	status = get_fence_status(lwis_fence);
 	read_len = len - copy_to_user((void __user *)user_buffer, (void *)&status + *offset, len);

 	return read_len;
 }

 /*
  *  lwis_fence_write_status: Signal fence with the error code from user
  */
 static ssize_t lwis_fence_write_status(struct file *fp, const char __user *user_buffer, size_t len,
 				       loff_t *offset)
 {
 	int ret = 0;
 	int status = 0;
 	struct lwis_fence *lwis_fence = fp->private_data;

 	if (!lwis_fence) {
 		return -EFAULT;
 	}

 	if (len != sizeof(lwis_fence->status)) {
 		dev_err(lwis_fence->lwis_top_dev->dev,
 			"Signal lwis_fence fd-%d with incorrect buffer length\n", lwis_fence->fd);
 		return -EINVAL;
 	}

 	/* Set lwis_fence's status if not signaled */
 	len = len - copy_from_user(&status, (void __user *)user_buffer, len);
 	ret = lwis_fence_signal(lwis_fence, status);
 	if (ret) {
 		return ret;
 	}

 	return len;
 }

 int lwis_fence_signal(struct lwis_fence *lwis_fence, int status)
 {
 	unsigned long flags;
 	struct lwis_fence_trigger_transaction_list *tx_list;
 	/* Temporary vars for hash table traversal */
 	struct hlist_node *n;
 	int i;

 	if (!lwis_fence) {
 		return -EFAULT;
 	}

 	spin_lock_irqsave(&lwis_fence->lock, flags);

 	if (lwis_fence->status != LWIS_FENCE_STATUS_NOT_SIGNALED) {
 		/* Return error if fence is already signaled */
 		dev_err(lwis_fence->lwis_top_dev->dev,
 			"Cannot signal a lwis_fence fd-%d already signaled, status is %d\n",
 			lwis_fence->fd, lwis_fence->status);
 		spin_unlock_irqrestore(&lwis_fence->lock, flags);
 		return -EINVAL;
 	}
 	lwis_fence->status = status;
 	spin_unlock_irqrestore(&lwis_fence->lock, flags);

 	wake_up_interruptible(&lwis_fence->status_wait_queue);

 	hash_for_each_safe (lwis_fence->transaction_list, i, n, tx_list, node) {
 		hash_del(&tx_list->node);
 		lwis_transaction_fence_trigger(tx_list->owner, lwis_fence, &tx_list->list);
 		if (!list_empty(&tx_list->list)) {
 			dev_err(lwis_fence->lwis_top_dev->dev,
 				"Fail to trigger all transactions\n");
 		}
 		kfree(tx_list);
 	}

 	return 0;
 }

 /*
  *  lwis_fence_poll: Poll status function of LWIS fence
  */
 static unsigned int lwis_fence_poll(struct file *fp, poll_table *wait)
 {
 	unsigned long flags;
 	int status = 0;
 	struct lwis_fence *lwis_fence = fp->private_data;
 	if (!lwis_fence) {
 		return POLLERR;
 	}

 	poll_wait(fp, &lwis_fence->status_wait_queue, wait);

 	spin_lock_irqsave(&lwis_fence->lock, flags);
 	status = lwis_fence->status;
 	spin_unlock_irqrestore(&lwis_fence->lock, flags);

 	/* Check if the fence is already signaled */
 	if (status != LWIS_FENCE_STATUS_NOT_SIGNALED) {
 		return POLLIN;
 	}

 	return 0;
 }

 int lwis_fence_create(struct lwis_device *lwis_dev)
 {
 	int fd_or_err;
 	struct lwis_fence *new_fence;

 	/* Allocate a new instance of lwis_fence struct */
 	new_fence = kmalloc(sizeof(struct lwis_fence), GFP_ATOMIC);
 	if (!new_fence) {
 		return -ENOMEM;
 	}

 	/* Open a new fd for the new fence */
 	fd_or_err =
 		anon_inode_getfd("lwis_fence_file", &fence_file_ops, new_fence, O_RDWR | O_CLOEXEC);
 	if (fd_or_err < 0) {
 		kfree(new_fence);
 		dev_err(lwis_dev->dev, "Failed to create a new file instance for lwis_fence\n");
 		return fd_or_err;
 	}

 	new_fence->fd = fd_or_err;
 	new_fence->lwis_top_dev = lwis_dev->top_dev;
 	new_fence->status = LWIS_FENCE_STATUS_NOT_SIGNALED;
 	spin_lock_init(&new_fence->lock);
 	init_waitqueue_head(&new_fence->status_wait_queue);
 	if (lwis_fence_debug) {
 		dev_info(lwis_dev->dev, "lwis_fence created new LWIS fence fd: %d", new_fence->fd);
 	}
 	return fd_or_err;
 }

 static struct lwis_fence_trigger_transaction_list *transaction_list_find(struct lwis_fence *fence,
 									 struct lwis_client *owner)
 {
 	int hash_key = HASH_CLIENT(owner);
 	struct lwis_fence_trigger_transaction_list *tx_list;
 	hash_for_each_possible (fence->transaction_list, tx_list, node, hash_key) {
 		if (tx_list->owner == owner) {
 			return tx_list;
 		}
 	}
 	return NULL;
 }

 static struct lwis_fence_trigger_transaction_list *
 transaction_list_create(struct lwis_fence *fence, struct lwis_client *owner)
 {
 	struct lwis_fence_trigger_transaction_list *tx_list =
 		kmalloc(sizeof(struct lwis_fence_trigger_transaction_list), GFP_ATOMIC);
 	if (!tx_list) {
 		return NULL;
 	}
 	tx_list->owner = owner;
 	INIT_LIST_HEAD(&tx_list->list);
 	hash_add(fence->transaction_list, &tx_list->node, HASH_CLIENT(owner));
 	return tx_list;
 }

 static struct lwis_fence_trigger_transaction_list *
 transaction_list_find_or_create(struct lwis_fence *fence, struct lwis_client *owner)
 {
 	struct lwis_fence_trigger_transaction_list *list = transaction_list_find(fence, owner);
 	return (list == NULL) ? transaction_list_create(fence, owner) : list;
 }

 static int lwis_trigger_fence_add_transaction(int fence_fd, struct lwis_client *client,
 					      struct lwis_transaction *transaction)
 {
 	unsigned long flags;
 	struct file *fp;
 	struct lwis_fence *lwis_fence;
 	struct lwis_pending_transaction_id *pending_transaction_id;
 	struct lwis_fence_trigger_transaction_list *tx_list;
 	int ret = 0;

 	if (transaction->num_trigger_fences >= LWIS_TRIGGER_NODES_MAX_NUM) {
 		dev_err(client->lwis_dev->dev,
 			"Invalid num_trigger_fences value in transaction %d\n", fence_fd);
 		return -EINVAL;
 	}

 	fp = fget(fence_fd);
 	if (fp == NULL) {
 		dev_err(client->lwis_dev->dev, "Failed to find lwis_fence with fd %d\n", fence_fd);
 		return -EBADF;
 	}
 	lwis_fence = fp->private_data;
 	if (lwis_fence->fd != fence_fd) {
 		fput(fp);
 		dev_err(client->lwis_dev->dev,
 			"Invalid lwis_fence with fd %d. Contains stale data \n", fence_fd);
 		return -EBADF;
 	}

 	pending_transaction_id = kmalloc(sizeof(struct lwis_pending_transaction_id), GFP_ATOMIC);
 	if (!pending_transaction_id) {
 		fput(fp);
 		return -ENOMEM;
 	}
 	pending_transaction_id->id = transaction->info.id;

 	spin_lock_irqsave(&lwis_fence->lock, flags);
 	if (lwis_fence->status == LWIS_FENCE_STATUS_NOT_SIGNALED) {
 		transaction->trigger_fence_fps[transaction->num_trigger_fences++] = fp;
 		tx_list = transaction_list_find_or_create(lwis_fence, client);
 		list_add(&pending_transaction_id->list_node, &tx_list->list);
 		if (lwis_fence_debug) {
 			dev_info(client->lwis_dev->dev,
 				 "lwis_fence transaction id %llu added to its trigger fence fd %d ",
 				 transaction->info.id, lwis_fence->fd);
 		}
 	} else {
 		kfree(pending_transaction_id);
 		if (lwis_fence_debug) {
 			dev_info(
 				client->lwis_dev->dev,
 				"lwis_fence fd-%d not added to transaction id %llu, fence already signaled with error code %d \n",
 				fence_fd, transaction->info.id, lwis_fence->status);
 		}
 		if (!transaction->info.is_level_triggered) {
 			/* If level triggering is disabled, return an error. */
 			fput(fp);
 			ret = -EINVAL;
 		} else {
 			transaction->trigger_fence_fps[transaction->num_trigger_fences++] = fp;
 			/* If the transaction's trigger_condition evaluates to true, queue the
 			 * transaction to be executed immediately.
 			 */
 			if (lwis_fence_triggered_condition_ready(transaction, lwis_fence->status)) {
 				if (lwis_fence->status != 0) {
 					transaction->resp->error_code = -ECANCELED;
 				}
 				transaction->queue_immediately = true;
 			}
 		}
 	}
 	spin_unlock_irqrestore(&lwis_fence->lock, flags);

 	return ret;
 }

 bool lwis_triggered_by_condition(struct lwis_transaction *transaction)
 {
 	return (transaction->info.trigger_condition.num_nodes > 0);
 }

 bool lwis_event_triggered_condition_ready(struct lwis_transaction *transaction,
 					  struct lwis_transaction *weak_transaction,
 					  int64_t event_id, int64_t event_counter)
 {
 	int32_t operator_type;
 	size_t all_signaled;
 	struct lwis_transaction_info_v2 *info = &transaction->info;
 	int i;
 	struct lwis_fence *lwis_fence;
 	bool is_node_signaled = false;

 	operator_type = info->trigger_condition.operator_type;
 	all_signaled = info->trigger_condition.num_nodes;

 	/*
 	 * Three scenarios to consider a node signaled:
 	 * 1) Event ID and event counter match,
 	 * 2) Event ID match, event counter not specified but precondition fence signaled, or,
 	 * 3) Event ID match, event counter and precondition fence not specified.
 	 */
 	for (i = 0; i < info->trigger_condition.num_nodes; i++) {
 		is_node_signaled = false;
 		if (info->trigger_condition.trigger_nodes[i].type == LWIS_TRIGGER_EVENT &&
 		    info->trigger_condition.trigger_nodes[i].event.id == event_id) {
 			if (info->trigger_condition.trigger_nodes[i].event.counter ==
 				    event_counter ||
 			    (info->trigger_condition.trigger_nodes[i].event.counter ==
 				     LWIS_EVENT_COUNTER_ON_NEXT_OCCURRENCE &&
 			     weak_transaction->precondition_fence_fp == NULL)) {
 				is_node_signaled = true;
 			} else if (info->trigger_condition.trigger_nodes[i].event.counter ==
 				   LWIS_EVENT_COUNTER_ON_NEXT_OCCURRENCE) {
 				lwis_fence = weak_transaction->precondition_fence_fp->private_data;
 				if (lwis_fence != NULL && get_fence_status(lwis_fence) == 0) {
 					is_node_signaled = true;
 					if (lwis_fence_debug) {
 						pr_info("TransactionId %lld: event 0x%llx (%lld), precondition fence %d signaled",
 							info->id, event_id, event_counter,
 							info->trigger_condition.trigger_nodes[i]
 								.event.precondition_fence_fd);
 					}
 				} else {
 					if (lwis_fence_debug) {
 						pr_info("TransactionId %lld: event 0x%llx (%lld), precondition fence %d NOT signaled yet",
 							info->id, event_id, event_counter,
 							info->trigger_condition.trigger_nodes[i]
 								.event.precondition_fence_fd);
 					}
 				}
 			}

 			if (is_node_signaled) {
 				transaction->signaled_count++;
 				list_del(&weak_transaction->event_list_node);
 				if (weak_transaction->precondition_fence_fp) {
 					fput(weak_transaction->precondition_fence_fp);
 				}
 				kfree(weak_transaction);
 				/* The break here assumes that this event ID only appears once in
 				   the trigger expression. Might need to revisit this. */
 				break;
 			}
 		}
 	}

 	if (i >= info->trigger_condition.num_nodes) {
 		/* No event counter is matched */
 		return false;
 	}

 	if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_AND &&
 	    transaction->signaled_count == all_signaled) {
 		return true;
 	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_OR) {
 		return true;
 	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_NONE) {
 		return true;
 	}

 	return false;
 }

 bool lwis_fence_triggered_condition_ready(struct lwis_transaction *transaction, int fence_status)
 {
 	int32_t operator_type;
 	size_t all_signaled;

 	operator_type = transaction->info.trigger_condition.operator_type;
 	all_signaled = transaction->info.trigger_condition.num_nodes;

 	transaction->signaled_count++;
 	if ((operator_type == LWIS_TRIGGER_NODE_OPERATOR_AND ||
 	     operator_type == LWIS_TRIGGER_NODE_OPERATOR_OR) &&
 	    transaction->signaled_count == all_signaled) {
 		return true;
 	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_AND && fence_status != 0) {
 		/*
 		   This condition is ready to cancel transaction as long as there is
 		   an error condition from fence with operator type "AND".
 		   No matter whether all condition nodes are signaled.
 		*/
 		return true;
 	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_OR && fence_status == 0) {
 		return true;
 	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_NONE) {
 		return true;
 	}

 	return false;
 }

 int lwis_parse_trigger_condition(struct lwis_client *client, struct lwis_transaction *transaction)
 {
 	struct lwis_transaction_info_v2 *info;
 	struct lwis_device *lwis_dev;
 	int i, ret;

 	if (!transaction || !client) {
 		return -EINVAL;
 	}

 	info = &transaction->info;
 	lwis_dev = client->lwis_dev;

 	if (info->trigger_condition.num_nodes > LWIS_TRIGGER_NODES_MAX_NUM) {
 		dev_err(lwis_dev->dev,
 			"Trigger condition contains %lu node, more than the limit of %d\n",
 			info->trigger_condition.num_nodes, LWIS_TRIGGER_NODES_MAX_NUM);
 		return -EINVAL;
 	}

 	for (i = 0; i < info->trigger_condition.num_nodes; i++) {
 		if (info->trigger_condition.trigger_nodes[i].type == LWIS_TRIGGER_EVENT) {
 			ret = lwis_trigger_event_add_weak_transaction(
 				client, info->id, info->trigger_condition.trigger_nodes[i].event.id,
 				info->trigger_condition.trigger_nodes[i]
 					.event.precondition_fence_fd);
 		} else {
 			ret = lwis_trigger_fence_add_transaction(
 				info->trigger_condition.trigger_nodes[i].fence_fd, client,
 				transaction);
 		}
 		if (ret) {
 			return ret;
 		}
 	}

 	return 0;
 }

 int ioctl_lwis_fence_create(struct lwis_device *lwis_dev, int32_t __user *msg)
 {
 	int32_t fd_or_err;

 	fd_or_err = lwis_fence_create(lwis_dev);
 	if (fd_or_err < 0) {
 		return fd_or_err;
 	}

 	if (copy_to_user((void __user *)msg, &fd_or_err, sizeof(int32_t))) {
 		dev_err(lwis_dev->dev, "failed to copy to user\n");
 		return -EFAULT;
 	}

 	return 0;
 }

 int lwis_initialize_transaction_fences(struct lwis_client *client,
 				       struct lwis_transaction *transaction)
 {
 	struct lwis_transaction_info_v2 *info = &transaction->info;
 	struct lwis_device *lwis_dev = client->lwis_dev;
 	int i;
 	int fd_or_err;

 	if (!transaction || !client) {
 		return -EINVAL;
 	}

 	/* If triggered by trigger_condition */
 	if (lwis_triggered_by_condition(transaction)) {
 		/* Initialize all placeholder fences in the trigger_condition */
 		for (i = 0; i < info->trigger_condition.num_nodes; i++) {
 			if (info->trigger_condition.trigger_nodes[i].type ==
 			    LWIS_TRIGGER_FENCE_PLACEHOLDER) {
 				fd_or_err = lwis_fence_create(lwis_dev);
 				if (fd_or_err < 0) {
 					return fd_or_err;
 				}
 				info->trigger_condition.trigger_nodes[i].fence_fd = fd_or_err;
 			}
 		}
 	}

 	/* Initialize completion fence if one is requested */
 	if (info->completion_fence_fd == LWIS_CREATE_COMPLETION_FENCE) {
 		fd_or_err = lwis_fence_create(client->lwis_dev);
 		if (fd_or_err < 0) {
 			return fd_or_err;
 		}
 		info->completion_fence_fd = fd_or_err;
 	}

 	return 0;
 }

 int lwis_add_completion_fence(struct lwis_client *client, struct lwis_transaction *transaction)
 {
 	struct file *fp;
 	struct lwis_fence *lwis_fence;
 	struct lwis_fence_pending_signal *fence_pending_signal;
 	struct lwis_device *lwis_dev = client->lwis_dev;
 	int fence_fd = transaction->info.completion_fence_fd;

 	/* If completion fence is not requested, we can safely return */
 	if (fence_fd == LWIS_NO_COMPLETION_FENCE) {
 		return 0;
 	}

 	/* If completion fence is requested but not initialized, we cannot continue */
 	if (fence_fd == LWIS_CREATE_COMPLETION_FENCE) {
 		dev_err(lwis_dev->dev,
 			"Cannot add uninitialized completion fence to transaction\n");
 		return -EPERM;
 	}

 	fp = fget(fence_fd);
 	if (fp == NULL) {
 		dev_err(lwis_dev->dev, "Failed to find lwis_fence with fd %d\n", fence_fd);
 		return -EBADF;
 	}

 	lwis_fence = fp->private_data;
 	fence_pending_signal = lwis_fence_pending_signal_create(lwis_fence, fp);
 	if (fence_pending_signal == NULL) {
 		return -ENOMEM;
 	}
 	list_add(&fence_pending_signal->node, &transaction->completion_fence_list);
 	if (lwis_fence_debug) {
 		dev_info(client->lwis_dev->dev,
 			 "lwis_fence transaction id %llu add completion fence fd %d ",
 			 transaction->info.id, lwis_fence->fd);
 	}
 	return 0;
 }

 struct lwis_fence_pending_signal *lwis_fence_pending_signal_create(struct lwis_fence *fence,
 								   struct file *fp)
 {
 	struct lwis_fence_pending_signal *pending_fence_signal =
 		kmalloc(sizeof(struct lwis_fence_pending_signal), GFP_ATOMIC);
 	if (!pending_fence_signal) {
 		return NULL;
 	}
 	pending_fence_signal->fp = fp;
 	pending_fence_signal->fence = fence;
 	pending_fence_signal->pending_status = LWIS_FENCE_STATUS_NOT_SIGNALED;
 	return pending_fence_signal;
 }

 void lwis_fences_pending_signal_emit(struct lwis_device *lwis_device,
 				     struct list_head *pending_fences)
 {
 	int ret;
 	struct lwis_fence_pending_signal *pending_fence;
 	struct list_head *it_fence, *it_fence_tmp;

 	list_for_each_safe (it_fence, it_fence_tmp, pending_fences) {
 		pending_fence = list_entry(it_fence, struct lwis_fence_pending_signal, node);
 		ret = lwis_fence_signal(pending_fence->fence, pending_fence->pending_status);
 		if (ret) {
 			dev_err(lwis_device->dev, "Failed signaling fence with fd %d",
 				pending_fence->fence->fd);
 		}
 		list_del(&pending_fence->node);
 		fput(pending_fence->fp);
 		kfree(pending_fence);
 	}
 }

 void lwis_pending_fences_move_all(struct lwis_device *lwis_device,
 				  struct lwis_transaction *transaction,
 				  struct list_head *pending_fences, int error_code)
 {
 	struct lwis_fence_pending_signal *pending_fence, *temp;

 	/* For each fence in transaction's signal list, move to pending_fences for signaling */
 	list_for_each_entry_safe (pending_fence, temp, &transaction->completion_fence_list, node) {
 		pending_fence->pending_status = error_code;
 		list_move_tail(&pending_fence->node, pending_fences);
 	}
 }
	/*
	* Google LWIS Fence
	*
	* Copyright (c) 2022 Google, LLC
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License version 2 as
	* published by the Free Software Foundation.
	*/

	#include <linux/anon_inodes.h>
	#include <linux/file.h>
	#include <linux/mm.h>
	#include <linux/poll.h>

	#include "lwis_device_top.h"
	#include "lwis_commands.h"
	#include "lwis_fence.h"
	#include "lwis_transaction.h"

	#define HASH_CLIENT(x) hash_ptr(x, LWIS_CLIENTS_HASH_BITS)

	bool lwis_fence_debug;
	module_param(lwis_fence_debug, bool, 0644);

	static int lwis_fence_release(struct inode node, struct file fp);
	static ssize_t lwis_fence_get_status(struct file fp, char __user user_buffer, size_t len,
	loff_t *offset);
	static ssize_t lwis_fence_write_status(struct file fp, const char __user user_buffer, size_t len,
	loff_t *offset);
	static unsigned int lwis_fence_poll(struct file fp, poll_table wait);

	static const struct file_operations fence_file_ops = {
	.owner = THIS_MODULE,
	.release = lwis_fence_release,
	.read = lwis_fence_get_status,
	.write = lwis_fence_write_status,
	.poll = lwis_fence_poll,
	};

	static int get_fence_status(struct lwis_fence *lwis_fence)
	{
	int status;
	unsigned long flags;
	spin_lock_irqsave(&lwis_fence->lock, flags);
	status = lwis_fence->status;
	spin_unlock_irqrestore(&lwis_fence->lock, flags);
	return status;
	}

	/*
	* lwis_fence_release: Closing an instance of a LWIS fence
	*/
	static int lwis_fence_release(struct inode node, struct file fp)
	{
	struct lwis_fence *lwis_fence = fp->private_data;
	struct lwis_fence_trigger_transaction_list *tx_list;
	struct lwis_pending_transaction_id *transaction_id;
	/* Temporary vars for traversal */
	struct hlist_node *n;
	struct list_head it_tran, it_tran_tmp;
	int i;

	if (lwis_fence_debug) {
	dev_info(lwis_fence->lwis_top_dev->dev, "Releasing lwis_fence fd-%d",
	lwis_fence->fd);
	}

	if (lwis_fence->status == LWIS_FENCE_STATUS_NOT_SIGNALED) {
	dev_err(lwis_fence->lwis_top_dev->dev,
	"lwis_fence fd-%d release without being signaled", lwis_fence->fd);
	}

	if (!hash_empty(lwis_fence->transaction_list)) {
	hash_for_each_safe (lwis_fence->transaction_list, i, n, tx_list, node) {
	if (!list_empty(&tx_list->list)) {
	list_for_each_safe (it_tran, it_tran_tmp, &tx_list->list) {
	transaction_id =
	list_entry(it_tran,
	struct lwis_pending_transaction_id,
	list_node);
	list_del(&transaction_id->list_node);
	kfree(transaction_id);
	}
	}
	hash_del(&tx_list->node);
	kfree(tx_list);
	}
	}

	kfree(lwis_fence);
	return 0;
	}

	/*
	* lwis_fence_get_status: Read the LWIS fence's status
	*/
	static ssize_t lwis_fence_get_status(struct file fp, char __user user_buffer, size_t len,
	loff_t *offset)
	{
	int status = 0;
	struct lwis_fence *lwis_fence = fp->private_data;
	int max_len, read_len;

	if (!lwis_fence) {
	return -EFAULT;
	}

	max_len = sizeof(status) - *offset;
	if (len > max_len) {
	len = max_len;
	}

	status = get_fence_status(lwis_fence);
	read_len = len - copy_to_user((void __user )user_buffer, (void )&status + *offset, len);

	return read_len;
	}

	/*
	* lwis_fence_write_status: Signal fence with the error code from user
	*/
	static ssize_t lwis_fence_write_status(struct file fp, const char __user user_buffer, size_t len,
	loff_t *offset)
	{
	int ret = 0;
	int status = 0;
	struct lwis_fence *lwis_fence = fp->private_data;

	if (!lwis_fence) {
	return -EFAULT;
	}

	if (len != sizeof(lwis_fence->status)) {
	dev_err(lwis_fence->lwis_top_dev->dev,
	"Signal lwis_fence fd-%d with incorrect buffer length\n", lwis_fence->fd);
	return -EINVAL;
	}

	/* Set lwis_fence's status if not signaled */
	len = len - copy_from_user(&status, (void __user *)user_buffer, len);
	ret = lwis_fence_signal(lwis_fence, status);
	if (ret) {
	return ret;
	}

	return len;
	}

	int lwis_fence_signal(struct lwis_fence *lwis_fence, int status)
	{
	unsigned long flags;
	struct lwis_fence_trigger_transaction_list *tx_list;
	/* Temporary vars for hash table traversal */
	struct hlist_node *n;
	int i;

	if (!lwis_fence) {
	return -EFAULT;
	}

	spin_lock_irqsave(&lwis_fence->lock, flags);

	if (lwis_fence->status != LWIS_FENCE_STATUS_NOT_SIGNALED) {
	/* Return error if fence is already signaled */
	dev_err(lwis_fence->lwis_top_dev->dev,
	"Cannot signal a lwis_fence fd-%d already signaled, status is %d\n",
	lwis_fence->fd, lwis_fence->status);
	spin_unlock_irqrestore(&lwis_fence->lock, flags);
	return -EINVAL;
	}
	lwis_fence->status = status;
	spin_unlock_irqrestore(&lwis_fence->lock, flags);

	wake_up_interruptible(&lwis_fence->status_wait_queue);

	hash_for_each_safe (lwis_fence->transaction_list, i, n, tx_list, node) {
	hash_del(&tx_list->node);
	lwis_transaction_fence_trigger(tx_list->owner, lwis_fence, &tx_list->list);
	if (!list_empty(&tx_list->list)) {
	dev_err(lwis_fence->lwis_top_dev->dev,
	"Fail to trigger all transactions\n");
	}
	kfree(tx_list);
	}

	return 0;
	}

	/*
	* lwis_fence_poll: Poll status function of LWIS fence
	*/
	static unsigned int lwis_fence_poll(struct file fp, poll_table wait)
	{
	unsigned long flags;
	int status = 0;
	struct lwis_fence *lwis_fence = fp->private_data;
	if (!lwis_fence) {
	return POLLERR;
	}

	poll_wait(fp, &lwis_fence->status_wait_queue, wait);

	spin_lock_irqsave(&lwis_fence->lock, flags);
	status = lwis_fence->status;
	spin_unlock_irqrestore(&lwis_fence->lock, flags);

	/* Check if the fence is already signaled */
	if (status != LWIS_FENCE_STATUS_NOT_SIGNALED) {
	return POLLIN;
	}

	return 0;
	}

	int lwis_fence_create(struct lwis_device *lwis_dev)
	{
	int fd_or_err;
	struct lwis_fence *new_fence;

	/* Allocate a new instance of lwis_fence struct */
	new_fence = kmalloc(sizeof(struct lwis_fence), GFP_ATOMIC);
	if (!new_fence) {
	return -ENOMEM;
	}

	/* Open a new fd for the new fence */
	fd_or_err =
	anon_inode_getfd("lwis_fence_file", &fence_file_ops, new_fence, O_RDWR \| O_CLOEXEC);
	if (fd_or_err < 0) {
	kfree(new_fence);
	dev_err(lwis_dev->dev, "Failed to create a new file instance for lwis_fence\n");
	return fd_or_err;
	}

	new_fence->fd = fd_or_err;
	new_fence->lwis_top_dev = lwis_dev->top_dev;
	new_fence->status = LWIS_FENCE_STATUS_NOT_SIGNALED;
	spin_lock_init(&new_fence->lock);
	init_waitqueue_head(&new_fence->status_wait_queue);
	if (lwis_fence_debug) {
	dev_info(lwis_dev->dev, "lwis_fence created new LWIS fence fd: %d", new_fence->fd);
	}
	return fd_or_err;
	}

	static struct lwis_fence_trigger_transaction_list transaction_list_find(struct lwis_fence fence,
	struct lwis_client *owner)
	{
	int hash_key = HASH_CLIENT(owner);
	struct lwis_fence_trigger_transaction_list *tx_list;
	hash_for_each_possible (fence->transaction_list, tx_list, node, hash_key) {
	if (tx_list->owner == owner) {
	return tx_list;
	}
	}
	return NULL;
	}

	static struct lwis_fence_trigger_transaction_list *
	transaction_list_create(struct lwis_fence fence, struct lwis_client owner)
	{
	struct lwis_fence_trigger_transaction_list *tx_list =
	kmalloc(sizeof(struct lwis_fence_trigger_transaction_list), GFP_ATOMIC);
	if (!tx_list) {
	return NULL;
	}
	tx_list->owner = owner;
	INIT_LIST_HEAD(&tx_list->list);
	hash_add(fence->transaction_list, &tx_list->node, HASH_CLIENT(owner));
	return tx_list;
	}

	static struct lwis_fence_trigger_transaction_list *
	transaction_list_find_or_create(struct lwis_fence fence, struct lwis_client owner)
	{
	struct lwis_fence_trigger_transaction_list *list = transaction_list_find(fence, owner);
	return (list == NULL) ? transaction_list_create(fence, owner) : list;
	}

	static int lwis_trigger_fence_add_transaction(int fence_fd, struct lwis_client *client,
	struct lwis_transaction *transaction)
	{
	unsigned long flags;
	struct file *fp;
	struct lwis_fence *lwis_fence;
	struct lwis_pending_transaction_id *pending_transaction_id;
	struct lwis_fence_trigger_transaction_list *tx_list;
	int ret = 0;

	if (transaction->num_trigger_fences >= LWIS_TRIGGER_NODES_MAX_NUM) {
	dev_err(client->lwis_dev->dev,
	"Invalid num_trigger_fences value in transaction %d\n", fence_fd);
	return -EINVAL;
	}

	fp = fget(fence_fd);
	if (fp == NULL) {
	dev_err(client->lwis_dev->dev, "Failed to find lwis_fence with fd %d\n", fence_fd);
	return -EBADF;
	}
	lwis_fence = fp->private_data;
	if (lwis_fence->fd != fence_fd) {
	fput(fp);
	dev_err(client->lwis_dev->dev,
	"Invalid lwis_fence with fd %d. Contains stale data \n", fence_fd);
	return -EBADF;
	}

	pending_transaction_id = kmalloc(sizeof(struct lwis_pending_transaction_id), GFP_ATOMIC);
	if (!pending_transaction_id) {
	fput(fp);
	return -ENOMEM;
	}
	pending_transaction_id->id = transaction->info.id;

	spin_lock_irqsave(&lwis_fence->lock, flags);
	if (lwis_fence->status == LWIS_FENCE_STATUS_NOT_SIGNALED) {
	transaction->trigger_fence_fps[transaction->num_trigger_fences++] = fp;
	tx_list = transaction_list_find_or_create(lwis_fence, client);
	list_add(&pending_transaction_id->list_node, &tx_list->list);
	if (lwis_fence_debug) {
	dev_info(client->lwis_dev->dev,
	"lwis_fence transaction id %llu added to its trigger fence fd %d ",
	transaction->info.id, lwis_fence->fd);
	}
	} else {
	kfree(pending_transaction_id);
	if (lwis_fence_debug) {
	dev_info(
	client->lwis_dev->dev,
	"lwis_fence fd-%d not added to transaction id %llu, fence already signaled with error code %d \n",
	fence_fd, transaction->info.id, lwis_fence->status);
	}
	if (!transaction->info.is_level_triggered) {
	/* If level triggering is disabled, return an error. */
	fput(fp);
	ret = -EINVAL;
	} else {
	transaction->trigger_fence_fps[transaction->num_trigger_fences++] = fp;
	/* If the transaction's trigger_condition evaluates to true, queue the
	* transaction to be executed immediately.
	*/
	if (lwis_fence_triggered_condition_ready(transaction, lwis_fence->status)) {
	if (lwis_fence->status != 0) {
	transaction->resp->error_code = -ECANCELED;
	}
	transaction->queue_immediately = true;
	}
	}
	}
	spin_unlock_irqrestore(&lwis_fence->lock, flags);

	return ret;
	}

	bool lwis_triggered_by_condition(struct lwis_transaction *transaction)
	{
	return (transaction->info.trigger_condition.num_nodes > 0);
	}

	bool lwis_event_triggered_condition_ready(struct lwis_transaction *transaction,
	struct lwis_transaction *weak_transaction,
	int64_t event_id, int64_t event_counter)
	{
	int32_t operator_type;
	size_t all_signaled;
	struct lwis_transaction_info_v2 *info = &transaction->info;
	int i;
	struct lwis_fence *lwis_fence;
	bool is_node_signaled = false;

	operator_type = info->trigger_condition.operator_type;
	all_signaled = info->trigger_condition.num_nodes;

	/*
	* Three scenarios to consider a node signaled:
	* 1) Event ID and event counter match,
	* 2) Event ID match, event counter not specified but precondition fence signaled, or,
	* 3) Event ID match, event counter and precondition fence not specified.
	*/
	for (i = 0; i < info->trigger_condition.num_nodes; i++) {
	is_node_signaled = false;
	if (info->trigger_condition.trigger_nodes[i].type == LWIS_TRIGGER_EVENT &&
	info->trigger_condition.trigger_nodes[i].event.id == event_id) {
	if (info->trigger_condition.trigger_nodes[i].event.counter ==
	event_counter \|\|
	(info->trigger_condition.trigger_nodes[i].event.counter ==
	LWIS_EVENT_COUNTER_ON_NEXT_OCCURRENCE &&
	weak_transaction->precondition_fence_fp == NULL)) {
	is_node_signaled = true;
	} else if (info->trigger_condition.trigger_nodes[i].event.counter ==
	LWIS_EVENT_COUNTER_ON_NEXT_OCCURRENCE) {
	lwis_fence = weak_transaction->precondition_fence_fp->private_data;
	if (lwis_fence != NULL && get_fence_status(lwis_fence) == 0) {
	is_node_signaled = true;
	if (lwis_fence_debug) {
	pr_info("TransactionId %lld: event 0x%llx (%lld), precondition fence %d signaled",
	info->id, event_id, event_counter,
	info->trigger_condition.trigger_nodes[i]
	.event.precondition_fence_fd);
	}
	} else {
	if (lwis_fence_debug) {
	pr_info("TransactionId %lld: event 0x%llx (%lld), precondition fence %d NOT signaled yet",
	info->id, event_id, event_counter,
	info->trigger_condition.trigger_nodes[i]
	.event.precondition_fence_fd);
	}
	}
	}

	if (is_node_signaled) {
	transaction->signaled_count++;
	list_del(&weak_transaction->event_list_node);
	if (weak_transaction->precondition_fence_fp) {
	fput(weak_transaction->precondition_fence_fp);
	}
	kfree(weak_transaction);
	/* The break here assumes that this event ID only appears once in
	the trigger expression. Might need to revisit this. */
	break;
	}
	}
	}

	if (i >= info->trigger_condition.num_nodes) {
	/* No event counter is matched */
	return false;
	}

	if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_AND &&
	transaction->signaled_count == all_signaled) {
	return true;
	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_OR) {
	return true;
	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_NONE) {
	return true;
	}

	return false;
	}

	bool lwis_fence_triggered_condition_ready(struct lwis_transaction *transaction, int fence_status)
	{
	int32_t operator_type;
	size_t all_signaled;

	operator_type = transaction->info.trigger_condition.operator_type;
	all_signaled = transaction->info.trigger_condition.num_nodes;

	transaction->signaled_count++;
	if ((operator_type == LWIS_TRIGGER_NODE_OPERATOR_AND \|\|
	operator_type == LWIS_TRIGGER_NODE_OPERATOR_OR) &&
	transaction->signaled_count == all_signaled) {
	return true;
	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_AND && fence_status != 0) {
	/*
	This condition is ready to cancel transaction as long as there is
	an error condition from fence with operator type "AND".
	No matter whether all condition nodes are signaled.
	*/
	return true;
	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_OR && fence_status == 0) {
	return true;
	} else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_NONE) {
	return true;
	}

	return false;
	}

	int lwis_parse_trigger_condition(struct lwis_client client, struct lwis_transaction transaction)
	{
	struct lwis_transaction_info_v2 *info;
	struct lwis_device *lwis_dev;
	int i, ret;

	if (!transaction \|\| !client) {
	return -EINVAL;
	}

	info = &transaction->info;
	lwis_dev = client->lwis_dev;

	if (info->trigger_condition.num_nodes > LWIS_TRIGGER_NODES_MAX_NUM) {
	dev_err(lwis_dev->dev,
	"Trigger condition contains %lu node, more than the limit of %d\n",
	info->trigger_condition.num_nodes, LWIS_TRIGGER_NODES_MAX_NUM);
	return -EINVAL;
	}

	for (i = 0; i < info->trigger_condition.num_nodes; i++) {
	if (info->trigger_condition.trigger_nodes[i].type == LWIS_TRIGGER_EVENT) {
	ret = lwis_trigger_event_add_weak_transaction(
	client, info->id, info->trigger_condition.trigger_nodes[i].event.id,
	info->trigger_condition.trigger_nodes[i]
	.event.precondition_fence_fd);
	} else {
	ret = lwis_trigger_fence_add_transaction(
	info->trigger_condition.trigger_nodes[i].fence_fd, client,
	transaction);
	}
	if (ret) {
	return ret;
	}
	}

	return 0;
	}

	int ioctl_lwis_fence_create(struct lwis_device lwis_dev, int32_t __user msg)
	{
	int32_t fd_or_err;

	fd_or_err = lwis_fence_create(lwis_dev);
	if (fd_or_err < 0) {
	return fd_or_err;
	}

	if (copy_to_user((void __user *)msg, &fd_or_err, sizeof(int32_t))) {
	dev_err(lwis_dev->dev, "failed to copy to user\n");
	return -EFAULT;
	}

	return 0;
	}

	int lwis_initialize_transaction_fences(struct lwis_client *client,
	struct lwis_transaction *transaction)
	{
	struct lwis_transaction_info_v2 *info = &transaction->info;
	struct lwis_device *lwis_dev = client->lwis_dev;
	int i;
	int fd_or_err;

	if (!transaction \|\| !client) {
	return -EINVAL;
	}

	/* If triggered by trigger_condition */
	if (lwis_triggered_by_condition(transaction)) {
	/* Initialize all placeholder fences in the trigger_condition */
	for (i = 0; i < info->trigger_condition.num_nodes; i++) {
	if (info->trigger_condition.trigger_nodes[i].type ==
	LWIS_TRIGGER_FENCE_PLACEHOLDER) {
	fd_or_err = lwis_fence_create(lwis_dev);
	if (fd_or_err < 0) {
	return fd_or_err;
	}
	info->trigger_condition.trigger_nodes[i].fence_fd = fd_or_err;
	}
	}
	}

	/* Initialize completion fence if one is requested */
	if (info->completion_fence_fd == LWIS_CREATE_COMPLETION_FENCE) {
	fd_or_err = lwis_fence_create(client->lwis_dev);
	if (fd_or_err < 0) {
	return fd_or_err;
	}
	info->completion_fence_fd = fd_or_err;
	}

	return 0;
	}

	int lwis_add_completion_fence(struct lwis_client client, struct lwis_transaction transaction)
	{
	struct file *fp;
	struct lwis_fence *lwis_fence;
	struct lwis_fence_pending_signal *fence_pending_signal;
	struct lwis_device *lwis_dev = client->lwis_dev;
	int fence_fd = transaction->info.completion_fence_fd;

	/* If completion fence is not requested, we can safely return */
	if (fence_fd == LWIS_NO_COMPLETION_FENCE) {
	return 0;
	}

	/* If completion fence is requested but not initialized, we cannot continue */
	if (fence_fd == LWIS_CREATE_COMPLETION_FENCE) {
	dev_err(lwis_dev->dev,
	"Cannot add uninitialized completion fence to transaction\n");
	return -EPERM;
	}

	fp = fget(fence_fd);
	if (fp == NULL) {
	dev_err(lwis_dev->dev, "Failed to find lwis_fence with fd %d\n", fence_fd);
	return -EBADF;
	}

	lwis_fence = fp->private_data;
	fence_pending_signal = lwis_fence_pending_signal_create(lwis_fence, fp);
	if (fence_pending_signal == NULL) {
	return -ENOMEM;
	}
	list_add(&fence_pending_signal->node, &transaction->completion_fence_list);
	if (lwis_fence_debug) {
	dev_info(client->lwis_dev->dev,
	"lwis_fence transaction id %llu add completion fence fd %d ",
	transaction->info.id, lwis_fence->fd);
	}
	return 0;
	}

	struct lwis_fence_pending_signal lwis_fence_pending_signal_create(struct lwis_fence fence,
	struct file *fp)
	{
	struct lwis_fence_pending_signal *pending_fence_signal =
	kmalloc(sizeof(struct lwis_fence_pending_signal), GFP_ATOMIC);
	if (!pending_fence_signal) {
	return NULL;
	}
	pending_fence_signal->fp = fp;
	pending_fence_signal->fence = fence;
	pending_fence_signal->pending_status = LWIS_FENCE_STATUS_NOT_SIGNALED;
	return pending_fence_signal;
	}

	void lwis_fences_pending_signal_emit(struct lwis_device *lwis_device,
	struct list_head *pending_fences)
	{
	int ret;
	struct lwis_fence_pending_signal *pending_fence;
	struct list_head it_fence, it_fence_tmp;

	list_for_each_safe (it_fence, it_fence_tmp, pending_fences) {
	pending_fence = list_entry(it_fence, struct lwis_fence_pending_signal, node);
	ret = lwis_fence_signal(pending_fence->fence, pending_fence->pending_status);
	if (ret) {
	dev_err(lwis_device->dev, "Failed signaling fence with fd %d",
	pending_fence->fence->fd);
	}
	list_del(&pending_fence->node);
	fput(pending_fence->fp);
	kfree(pending_fence);
	}
	}

	void lwis_pending_fences_move_all(struct lwis_device *lwis_device,
	struct lwis_transaction *transaction,
	struct list_head *pending_fences, int error_code)
	{
	struct lwis_fence_pending_signal pending_fence, temp;

	/* For each fence in transaction's signal list, move to pending_fences for signaling */
	list_for_each_entry_safe (pending_fence, temp, &transaction->completion_fence_list, node) {
	pending_fence->pending_status = error_code;
	list_move_tail(&pending_fence->node, pending_fences);
	}
	}