mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2024-11-22 03:59:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'iommu/iommufd/attach-handles' into iommu/next

Browse Source 
* iommu/iommufd/attach-handles:
  iommu: Extend domain attach group with handle support
  iommu: Add attach handle to struct iopf_group
  iommu: Remove sva handle list
  iommu: Introduce domain attachment handle
This commit is contained in:
Will Deacon 2024-07-12 16:57:42 +01:00
commit 74e54d532b
6 changed files with 238 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/dma/idxd/init.c
View File

@ -584,7 +584,7 @@ static int idxd_enable_system_pasid(struct idxd_device *idxd)
* DMA domain is owned by the driver, it should support all valid
* types such as DMA-FQ, identity, etc.
*/
ret = iommu_attach_device_pasid(domain, dev, pasid);
ret = iommu_attach_device_pasid(domain, dev, pasid, NULL);
if (ret) {
dev_err(dev, "failed to attach device pasid %d, domain type %d",
pasid, domain->type);

61
drivers/iommu/io-pgfault.c
View File

@ -59,30 +59,6 @@ void iopf_free_group(struct iopf_group *group)
}
EXPORT_SYMBOL_GPL(iopf_free_group);
static struct iommu_domain *get_domain_for_iopf(struct device *dev,
struct iommu_fault *fault)
{
struct iommu_domain *domain;
if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) {
domain = iommu_get_domain_for_dev_pasid(dev, fault->prm.pasid, 0);
if (IS_ERR(domain))
domain = NULL;
} else {
domain = iommu_get_domain_for_dev(dev);
}
if (!domain || !domain->iopf_handler) {
dev_warn_ratelimited(dev,
"iopf (pasid %d) without domain attached or handler installed\n",
fault->prm.pasid);
return NULL;
}
return domain;
}
/* Non-last request of a group. Postpone until the last one. */
static int report_partial_fault(struct iommu_fault_param *fault_param,
struct iommu_fault *fault)
@ -206,20 +182,51 @@ void iommu_report_device_fault(struct device *dev, struct iopf_fault *evt)
if (group == &abort_group)
goto err_abort;
group->domain = get_domain_for_iopf(dev, fault);
if (!group->domain)
if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) {
group->attach_handle = iommu_attach_handle_get(dev->iommu_group,
fault->prm.pasid,
0);
if (IS_ERR(group->attach_handle)) {
const struct iommu_ops *ops = dev_iommu_ops(dev);
if (!ops->user_pasid_table)
goto err_abort;
/*
* The iommu driver for this device supports user-
* managed PASID table. Therefore page faults for
* any PASID should go through the NESTING domain
* attached to the device RID.
*/
group->attach_handle =
iommu_attach_handle_get(dev->iommu_group,
IOMMU_NO_PASID,
IOMMU_DOMAIN_NESTED);
if (IS_ERR(group->attach_handle))
goto err_abort;
}
} else {
group->attach_handle =
iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0);
if (IS_ERR(group->attach_handle))
goto err_abort;
}
if (!group->attach_handle->domain->iopf_handler)
goto err_abort;
/*
* On success iopf_handler must call iopf_group_response() and
* iopf_free_group()
*/
if (group->domain->iopf_handler(group))
if (group->attach_handle->domain->iopf_handler(group))
goto err_abort;
return;
err_abort:
dev_warn_ratelimited(dev, "iopf with pasid %d aborted\n",
fault->prm.pasid);
iopf_group_response(group, IOMMU_PAGE_RESP_FAILURE);
if (group == &abort_group)
__iopf_free_group(group);

11
drivers/iommu/iommu-priv.h
View File

@ -35,4 +35,15 @@ void iommu_device_unregister_bus(struct iommu_device *iommu,
const struct bus_type *bus,
struct notifier_block *nb);
struct iommu_attach_handle *iommu_attach_handle_get(struct iommu_group *group,
ioasid_t pasid,
unsigned int type);
int iommu_attach_group_handle(struct iommu_domain *domain,
struct iommu_group *group,
struct iommu_attach_handle *handle);
void iommu_detach_group_handle(struct iommu_domain *domain,
struct iommu_group *group);
int iommu_replace_group_handle(struct iommu_group *group,
struct iommu_domain *new_domain,
struct iommu_attach_handle *handle);
#endif /* __LINUX_IOMMU_PRIV_H */

42
drivers/iommu/iommu-sva.c
View File

@ -43,7 +43,6 @@ static struct iommu_mm_data *iommu_alloc_mm_data(struct mm_struct *mm, struct de
}
iommu_mm->pasid = pasid;
INIT_LIST_HEAD(&iommu_mm->sva_domains);
INIT_LIST_HEAD(&iommu_mm->sva_handles);
/*
* Make sure the write to mm->iommu_mm is not reordered in front of
* initialization to iommu_mm fields. If it does, readers may see a
@ -71,11 +70,16 @@ static struct iommu_mm_data *iommu_alloc_mm_data(struct mm_struct *mm, struct de
*/
struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
{
struct iommu_group *group = dev->iommu_group;
struct iommu_attach_handle *attach_handle;
struct iommu_mm_data *iommu_mm;
struct iommu_domain *domain;
struct iommu_sva *handle;
int ret;
if (!group)
return ERR_PTR(-ENODEV);
mutex_lock(&iommu_sva_lock);
/* Allocate mm->pasid if necessary. */
@ -85,12 +89,22 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm
goto out_unlock;
}
list_for_each_entry(handle, &mm->iommu_mm->sva_handles, handle_item) {
if (handle->dev == dev) {
refcount_inc(&handle->users);
mutex_unlock(&iommu_sva_lock);
return handle;
/* A bond already exists, just take a reference`. */
attach_handle = iommu_attach_handle_get(group, iommu_mm->pasid, IOMMU_DOMAIN_SVA);
if (!IS_ERR(attach_handle)) {
handle = container_of(attach_handle, struct iommu_sva, handle);
if (attach_handle->domain->mm != mm) {
ret = -EBUSY;
goto out_unlock;
}
refcount_inc(&handle->users);
mutex_unlock(&iommu_sva_lock);
return handle;
}
if (PTR_ERR(attach_handle) != -ENOENT) {
ret = PTR_ERR(attach_handle);
goto out_unlock;
}
handle = kzalloc(sizeof(*handle), GFP_KERNEL);
@ -101,7 +115,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm
/* Search for an existing domain. */
list_for_each_entry(domain, &mm->iommu_mm->sva_domains, next) {
ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid,
&handle->handle);
if (!ret) {
domain->users++;
goto out;
@ -115,7 +130,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm
goto out_free_handle;
}
ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid,
&handle->handle);
if (ret)
goto out_free_domain;
domain->users = 1;
@ -123,10 +139,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm
out:
refcount_set(&handle->users, 1);
list_add(&handle->handle_item, &mm->iommu_mm->sva_handles);
mutex_unlock(&iommu_sva_lock);
handle->dev = dev;
handle->domain = domain;
return handle;
out_free_domain:
@ -149,7 +163,7 @@ EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
*/
void iommu_sva_unbind_device(struct iommu_sva *handle)
{
struct iommu_domain *domain = handle->domain;
struct iommu_domain *domain = handle->handle.domain;
struct iommu_mm_data *iommu_mm = domain->mm->iommu_mm;
struct device *dev = handle->dev;
@ -158,7 +172,6 @@ void iommu_sva_unbind_device(struct iommu_sva *handle)
mutex_unlock(&iommu_sva_lock);
return;
}
list_del(&handle->handle_item);
iommu_detach_device_pasid(domain, dev, iommu_mm->pasid);
if (--domain->users == 0) {
@ -172,7 +185,7 @@ EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
u32 iommu_sva_get_pasid(struct iommu_sva *handle)
{
struct iommu_domain *domain = handle->domain;
struct iommu_domain *domain = handle->handle.domain;
return mm_get_enqcmd_pasid(domain->mm);
}
@ -261,7 +274,8 @@ static void iommu_sva_handle_iopf(struct work_struct *work)
if (status != IOMMU_PAGE_RESP_SUCCESS)
break;
status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm);
status = iommu_sva_handle_mm(&iopf->fault,
group->attach_handle->domain->mm);
}
iopf_group_response(group, status);

185
drivers/iommu/iommu.c
View File

@ -3348,16 +3348,17 @@ static void __iommu_remove_group_pasid(struct iommu_group *group,
* @domain: the iommu domain.
* @dev: the attached device.
* @pasid: the pasid of the device.
* @handle: the attach handle.
*
* Return: 0 on success, or an error.
*/
int iommu_attach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid)
struct device *dev, ioasid_t pasid,
struct iommu_attach_handle *handle)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
struct group_device *device;
void *curr;
int ret;
if (!domain->ops->set_dev_pasid)
@ -3378,11 +3379,12 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
}
}
curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
if (curr) {
ret = xa_err(curr) ? : -EBUSY;
if (handle)
handle->domain = domain;
ret = xa_insert(&group->pasid_array, pasid, handle, GFP_KERNEL);
if (ret)
goto out_unlock;
}
ret = __iommu_set_group_pasid(domain, group, pasid);
if (ret)
@ -3410,46 +3412,11 @@ void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
mutex_lock(&group->mutex);
__iommu_remove_group_pasid(group, pasid, domain);
WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
xa_erase(&group->pasid_array, pasid);
mutex_unlock(&group->mutex);
}
EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
/*
* iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
* @dev: the queried device
* @pasid: the pasid of the device
* @type: matched domain type, 0 for any match
*
* This is a variant of iommu_get_domain_for_dev(). It returns the existing
* domain attached to pasid of a device. Callers must hold a lock around this
* function, and both iommu_attach/detach_dev_pasid() whenever a domain of
* type is being manipulated. This API does not internally resolve races with
* attach/detach.
*
* Return: attached domain on success, NULL otherwise.
*/
struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
ioasid_t pasid,
unsigned int type)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
struct iommu_domain *domain;
if (!group)
return NULL;
xa_lock(&group->pasid_array);
domain = xa_load(&group->pasid_array, pasid);
if (type && domain && domain->type != type)
domain = ERR_PTR(-EBUSY);
xa_unlock(&group->pasid_array);
return domain;
}
EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
ioasid_t iommu_alloc_global_pasid(struct device *dev)
{
int ret;
@ -3476,3 +3443,137 @@ void iommu_free_global_pasid(ioasid_t pasid)
ida_free(&iommu_global_pasid_ida, pasid);
}
EXPORT_SYMBOL_GPL(iommu_free_global_pasid);
/**
* iommu_attach_handle_get - Return the attach handle
* @group: the iommu group that domain was attached to
* @pasid: the pasid within the group
* @type: matched domain type, 0 for any match
*
* Return handle or ERR_PTR(-ENOENT) on none, ERR_PTR(-EBUSY) on mismatch.
*
* Return the attach handle to the caller. The life cycle of an iommu attach
* handle is from the time when the domain is attached to the time when the
* domain is detached. Callers are required to synchronize the call of
* iommu_attach_handle_get() with domain attachment and detachment. The attach
* handle can only be used during its life cycle.
*/
struct iommu_attach_handle *
iommu_attach_handle_get(struct iommu_group *group, ioasid_t pasid, unsigned int type)
{
struct iommu_attach_handle *handle;
xa_lock(&group->pasid_array);
handle = xa_load(&group->pasid_array, pasid);
if (!handle)
handle = ERR_PTR(-ENOENT);
else if (type && handle->domain->type != type)
handle = ERR_PTR(-EBUSY);
xa_unlock(&group->pasid_array);
return handle;
}
EXPORT_SYMBOL_NS_GPL(iommu_attach_handle_get, IOMMUFD_INTERNAL);
/**
* iommu_attach_group_handle - Attach an IOMMU domain to an IOMMU group
* @domain: IOMMU domain to attach
* @group: IOMMU group that will be attached
* @handle: attach handle
*
* Returns 0 on success and error code on failure.
*
* This is a variant of iommu_attach_group(). It allows the caller to provide
* an attach handle and use it when the domain is attached. This is currently
* used by IOMMUFD to deliver the I/O page faults.
*/
int iommu_attach_group_handle(struct iommu_domain *domain,
struct iommu_group *group,
struct iommu_attach_handle *handle)
{
int ret;
if (handle)
handle->domain = domain;
mutex_lock(&group->mutex);
ret = xa_insert(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
if (ret)
goto err_unlock;
ret = __iommu_attach_group(domain, group);
if (ret)
goto err_erase;
mutex_unlock(&group->mutex);
return 0;
err_erase:
xa_erase(&group->pasid_array, IOMMU_NO_PASID);
err_unlock:
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_NS_GPL(iommu_attach_group_handle, IOMMUFD_INTERNAL);
/**
* iommu_detach_group_handle - Detach an IOMMU domain from an IOMMU group
* @domain: IOMMU domain to attach
* @group: IOMMU group that will be attached
*
* Detach the specified IOMMU domain from the specified IOMMU group.
* It must be used in conjunction with iommu_attach_group_handle().
*/
void iommu_detach_group_handle(struct iommu_domain *domain,
struct iommu_group *group)
{
mutex_lock(&group->mutex);
__iommu_group_set_core_domain(group);
xa_erase(&group->pasid_array, IOMMU_NO_PASID);
mutex_unlock(&group->mutex);
}
EXPORT_SYMBOL_NS_GPL(iommu_detach_group_handle, IOMMUFD_INTERNAL);
/**
* iommu_replace_group_handle - replace the domain that a group is attached to
* @group: IOMMU group that will be attached to the new domain
* @new_domain: new IOMMU domain to replace with
* @handle: attach handle
*
* This is a variant of iommu_group_replace_domain(). It allows the caller to
* provide an attach handle for the new domain and use it when the domain is
* attached.
*/
int iommu_replace_group_handle(struct iommu_group *group,
struct iommu_domain *new_domain,
struct iommu_attach_handle *handle)
{
void *curr;
int ret;
if (!new_domain)
return -EINVAL;
mutex_lock(&group->mutex);
if (handle) {
ret = xa_reserve(&group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);
if (ret)
goto err_unlock;
}
ret = __iommu_group_set_domain(group, new_domain);
if (ret)
goto err_release;
curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
WARN_ON(xa_is_err(curr));
mutex_unlock(&group->mutex);
return 0;
err_release:
xa_release(&group->pasid_array, IOMMU_NO_PASID);
err_unlock:
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, IOMMUFD_INTERNAL);

37
include/linux/iommu.h
View File

@ -127,7 +127,7 @@ struct iopf_group {
/* list node for iommu_fault_param::faults */
struct list_head pending_node;
struct work_struct work;
struct iommu_domain *domain;
struct iommu_attach_handle *attach_handle;
/* The device's fault data parameter. */
struct iommu_fault_param *fault_param;
};
@ -547,6 +547,10 @@ static inline int __iommu_copy_struct_from_user_array(
* @default_domain: If not NULL this will always be set as the default domain.
* This should be an IDENTITY/BLOCKED/PLATFORM domain.
* Do not use in new drivers.
* @user_pasid_table: IOMMU driver supports user-managed PASID table. There is
* no user domain for each PASID and the I/O page faults are
* forwarded through the user domain attached to the device
* RID.
*/
struct iommu_ops {
bool (*capable)(struct device *dev, enum iommu_cap);
@ -590,6 +594,7 @@ struct iommu_ops {
struct iommu_domain *blocked_domain;
struct iommu_domain *release_domain;
struct iommu_domain *default_domain;
u8 user_pasid_table:1;
};
/**
@ -987,20 +992,28 @@ struct iommu_fwspec {
/* ATS is supported */
#define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0)
/*
* An iommu attach handle represents a relationship between an iommu domain
* and a PASID or RID of a device. It is allocated and managed by the component
* that manages the domain and is stored in the iommu group during the time the
* domain is attached.
*/
struct iommu_attach_handle {
struct iommu_domain *domain;
};
/**
* struct iommu_sva - handle to a device-mm bond
*/
struct iommu_sva {
struct iommu_attach_handle handle;
struct device *dev;
struct iommu_domain *domain;
struct list_head handle_item;
refcount_t users;
};
struct iommu_mm_data {
u32 pasid;
struct list_head sva_domains;
struct list_head sva_handles;
};
int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode);
@ -1048,12 +1061,10 @@ int iommu_device_claim_dma_owner(struct device *dev, void *owner);
void iommu_device_release_dma_owner(struct device *dev);
int iommu_attach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid);
struct device *dev, ioasid_t pasid,
struct iommu_attach_handle *handle);
void iommu_detach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid);
struct iommu_domain *
iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
unsigned int type);
ioasid_t iommu_alloc_global_pasid(struct device *dev);
void iommu_free_global_pasid(ioasid_t pasid);
#else /* CONFIG_IOMMU_API */
@ -1377,7 +1388,8 @@ static inline int iommu_device_claim_dma_owner(struct device *dev, void *owner)
}
static inline int iommu_attach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid)
struct device *dev, ioasid_t pasid,
struct iommu_attach_handle *handle)
{
return -ENODEV;
}
@ -1387,13 +1399,6 @@ static inline void iommu_detach_device_pasid(struct iommu_domain *domain,
{
}
static inline struct iommu_domain *
iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
unsigned int type)
{
return NULL;
}
static inline ioasid_t iommu_alloc_global_pasid(struct device *dev)
{
return IOMMU_PASID_INVALID;