1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
|
// SPDX-License-Identifier: GPL-2.0 OR MIT
/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
#include <linux/interrupt.h>
#include <linux/iopoll.h>
#include <linux/device.h>
#include "lima_device.h"
#include "lima_mmu.h"
#include "lima_vm.h"
#include "lima_regs.h"
#define mmu_write(reg, data) writel(data, ip->iomem + reg)
#define mmu_read(reg) readl(ip->iomem + reg)
#define lima_mmu_send_command(cmd, addr, val, cond) \
({ \
int __ret; \
\
mmu_write(LIMA_MMU_COMMAND, cmd); \
__ret = readl_poll_timeout(ip->iomem + (addr), val, \
cond, 0, 100); \
if (__ret) \
dev_err(dev->dev, \
"mmu command %x timeout\n", cmd); \
__ret; \
})
static irqreturn_t lima_mmu_irq_handler(int irq, void *data)
{
struct lima_ip *ip = data;
struct lima_device *dev = ip->dev;
u32 status = mmu_read(LIMA_MMU_INT_STATUS);
struct lima_sched_pipe *pipe;
/* for shared irq case */
if (!status)
return IRQ_NONE;
if (status & LIMA_MMU_INT_PAGE_FAULT) {
u32 fault = mmu_read(LIMA_MMU_PAGE_FAULT_ADDR);
dev_err(dev->dev, "mmu page fault at 0x%x from bus id %d of type %s on %s\n",
fault, LIMA_MMU_STATUS_BUS_ID(status),
status & LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE ? "write" : "read",
lima_ip_name(ip));
}
if (status & LIMA_MMU_INT_READ_BUS_ERROR)
dev_err(dev->dev, "mmu %s irq bus error\n", lima_ip_name(ip));
/* mask all interrupts before resume */
mmu_write(LIMA_MMU_INT_MASK, 0);
mmu_write(LIMA_MMU_INT_CLEAR, status);
pipe = dev->pipe + (ip->id == lima_ip_gpmmu ? lima_pipe_gp : lima_pipe_pp);
lima_sched_pipe_mmu_error(pipe);
return IRQ_HANDLED;
}
static int lima_mmu_hw_init(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
int err;
u32 v;
mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
LIMA_MMU_DTE_ADDR, v, v == 0);
if (err)
return err;
mmu_write(LIMA_MMU_INT_MASK,
LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
LIMA_MMU_STATUS, v,
v & LIMA_MMU_STATUS_PAGING_ENABLED);
}
int lima_mmu_resume(struct lima_ip *ip)
{
if (ip->id == lima_ip_ppmmu_bcast)
return 0;
return lima_mmu_hw_init(ip);
}
void lima_mmu_suspend(struct lima_ip *ip)
{
}
int lima_mmu_init(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
int err;
if (ip->id == lima_ip_ppmmu_bcast)
return 0;
mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
if (mmu_read(LIMA_MMU_DTE_ADDR) != 0xCAFEB000) {
dev_err(dev->dev, "mmu %s dte write test fail\n", lima_ip_name(ip));
return -EIO;
}
err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler,
IRQF_SHARED, lima_ip_name(ip), ip);
if (err) {
dev_err(dev->dev, "mmu %s fail to request irq\n", lima_ip_name(ip));
return err;
}
return lima_mmu_hw_init(ip);
}
void lima_mmu_fini(struct lima_ip *ip)
{
}
void lima_mmu_flush_tlb(struct lima_ip *ip)
{
mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
}
void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm)
{
struct lima_device *dev = ip->dev;
u32 v;
lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_STALL,
LIMA_MMU_STATUS, v,
v & LIMA_MMU_STATUS_STALL_ACTIVE);
mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
/* flush the TLB */
mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
lima_mmu_send_command(LIMA_MMU_COMMAND_DISABLE_STALL,
LIMA_MMU_STATUS, v,
!(v & LIMA_MMU_STATUS_STALL_ACTIVE));
}
void lima_mmu_page_fault_resume(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
u32 status = mmu_read(LIMA_MMU_STATUS);
u32 v;
if (status & LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE) {
dev_info(dev->dev, "mmu resume\n");
mmu_write(LIMA_MMU_INT_MASK, 0);
mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
LIMA_MMU_DTE_ADDR, v, v == 0);
mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
LIMA_MMU_STATUS, v,
v & LIMA_MMU_STATUS_PAGING_ENABLED);
}
}
|