/*
 * Copyright © 2020 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

#include <sys/ioctl.h>
#include <cairo.h>

#include "i915/gem_create.h"
#include "igt.h"
#include "igt_x86.h"
#include "intel_bufops.h"
#include "intel_mocs.h"
#include "intel_pat.h"
#include "xe/xe_ioctl.h"
#include "xe/xe_query.h"

/**
 * SECTION:intel_bufops
 * @short_description: Buffer operation on tiled surfaces
 * @title: Buffer operations
 * @include: igt.h
 *
 * # Buffer operations
 *
 * Intel GPU devices supports different set of tiled surfaces.
 * Checking each time what tile formats are supports is cumbersome and
 * error prone.
 *
 * Buffer operation (buf_ops) provide a wrapper to conditional code which
 * can be used without worrying of implementation details giving:
 * - copy linear to tiled buffer
 * - copy tiled buffer to linear
 *
 * Following code order should be used (linear is plain memory with some
 * image data):
 *
 * |[<!-- language="c" -->
 * struct buf_ops *bops;
 * struct intel_buf ibuf;
 * ...
 * bops = buf_ops_create(fd);
 * intel_buf_init(bops, &ibuf, 512, 512, 32, 64, I915_TILING_X, false);
 * ...
 * linear_to_intel_buf(bops, &ibuf, linear);
 * ...
 * intel_buf_to_linear(bops, &ibuf, linear);
 * ...
 * intel_buf_close(bops, &ibuf);
 * ...
 * buf_ops_destroy(bops);
 * ]|
 *
 * Calling buf_ops_create(fd) probes hardware capabilities (supported fences,
 * swizzling) and returns opaque pointer to buf_ops. From now on
 * intel_buf_to_linear() and linear_to_intel_buf() will choose appropriate
 * function to do the job.
 *
 * Note: bufops doesn't support SW tiling code yet.
 */

//#define BUFOPS_DEBUGGING
#ifdef BUFOPS_DEBUGGING
#define DEBUG(...) printf(__VA_ARGS__)
#define DEBUGFN() DEBUG("\t -> %s()\n", __FUNCTION__)
#else
#define DEBUG(...)
#define DEBUGFN()
#endif

#undef TILE_NONE
#undef TILE_X
#undef TILE_Y
#undef TILE_Yf
#undef TILE_Ys
#undef TILE_4
#undef TILE_64

#define TILE_DEF(x) (1 << (x))
#define TILE_NONE   TILE_DEF(I915_TILING_NONE)
#define TILE_X      TILE_DEF(I915_TILING_X)
#define TILE_Y      TILE_DEF(I915_TILING_Y)
#define TILE_Yf     TILE_DEF(I915_TILING_Yf)
#define TILE_Ys     TILE_DEF(I915_TILING_Ys)
#define TILE_4      TILE_DEF(I915_TILING_4)
#define TILE_64     TILE_DEF(I915_TILING_64)

#define CCS_OFFSET(buf) (buf->ccs[0].offset)
#define CCS_SIZE(gen, buf) \
	(intel_buf_ccs_width(gen, buf) * intel_buf_ccs_height(gen, buf))

typedef void (*bo_copy)(struct buf_ops *, struct intel_buf *, uint32_t *);

struct buf_ops {
	int fd;
	enum intel_driver driver;
	int gen_start;
	int gen_end;
	unsigned int intel_gen;
	uint32_t devid;
	uint32_t supported_tiles;
	uint32_t supported_hw_tiles;
	uint32_t swizzle_x;
	uint32_t swizzle_y;
	bo_copy linear_to;
	bo_copy linear_to_x;
	bo_copy linear_to_y;
	bo_copy linear_to_yf;
	bo_copy linear_to_ys;
	bo_copy linear_to_tile4;
	bo_copy to_linear;
	bo_copy x_to_linear;
	bo_copy y_to_linear;
	bo_copy yf_to_linear;
	bo_copy ys_to_linear;
	bo_copy tile4_to_linear;
};

static const char *tiling_str(uint32_t tiling)
{
	switch (tiling) {
	case I915_TILING_NONE: return "NONE";
	case I915_TILING_X:    return "X";
	case I915_TILING_Y:    return "Y";
	case I915_TILING_Yf:   return "Yf";
	case I915_TILING_Ys:   return "Ys";
	case I915_TILING_4:    return "4";
	case I915_TILING_64:   return "64";
	default:               return "UNKNOWN";
	}
}

static const char *bool_str(bool v)
{
	return v ? "yes" : "no";
}

static inline bool is_hw_tiling_supported(struct buf_ops *bops, uint32_t tiling)
{
	return bops->supported_hw_tiles & TILE_DEF(tiling);
}

static inline bool is_tiling_supported(struct buf_ops *bops, uint32_t tiling)
{
	return bops->supported_tiles & TILE_DEF(tiling);
}

static uint32_t get_stride(uint32_t devid, uint32_t tiling)
{
	uint32_t stride = 128;

	if (IS_915G(devid) || IS_915GM(devid) || tiling == I915_TILING_X)
		stride = 512;

	return stride;
}

static int __gem_get_tiling(int fd, struct drm_i915_gem_get_tiling *arg)
{
	int err;

	err = 0;
	if (igt_ioctl(fd, DRM_IOCTL_I915_GEM_GET_TILING, arg)) {
		err = -errno;
		igt_assume(err);
	}
	errno = 0;

	return err;
}

static bool __get_tiling(int fd, uint32_t handle, uint32_t *tiling,
			 uint32_t *swizzle, uint32_t *phys_swizzle)
{
	struct drm_i915_gem_get_tiling get_tiling = { .handle = handle };

	if (__gem_get_tiling(fd, &get_tiling) != 0)
		return false;

	*tiling = get_tiling.tiling_mode;
	*swizzle = get_tiling.swizzle_mode;
	*phys_swizzle = get_tiling.phys_swizzle_mode;
	igt_debug("buf tiling: %s, swizzle: %x, phys_swizzle: %x\n",
		  tiling_str(get_tiling.tiling_mode),
		  get_tiling.swizzle_mode,
		  get_tiling.phys_swizzle_mode);

	return true;
}

static int __set_tiling(int fd, uint32_t handle, uint32_t tiling,
			uint32_t stride,
			uint32_t *ret_tiling, uint32_t *ret_swizzle)
{
	struct drm_i915_gem_set_tiling st;

	memset(&st, 0, sizeof(st));
	do {

		int err;

		st.handle = handle;
		st.tiling_mode = tiling;
		st.stride = tiling ? stride : 0;

		err = 0;
		if (ioctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &st))
			err = -errno;
		errno = 0;
		if (err != -EINTR) {
			if (ret_tiling)
				*ret_tiling = st.tiling_mode;

			if (ret_swizzle)
				*ret_swizzle = st.swizzle_mode;

			return err;
		}
	} while (1);
}

static void set_hw_tiled(struct buf_ops *bops, struct intel_buf *buf)
{
	uint32_t ret_tiling, ret_swizzle;

	if (buf->tiling != I915_TILING_X && buf->tiling != I915_TILING_Y &&
	    buf->tiling != I915_TILING_4)
		return;

	if (!buf_ops_has_hw_fence(bops, buf->tiling)) {
		igt_debug("No HW fence for tiling: %d\n", buf->tiling);
		return;
	}

	igt_assert_eq(__set_tiling(bops->fd,
				   buf->handle, buf->tiling,
				   buf->surface[0].stride,
				   &ret_tiling, &ret_swizzle),
		      0);

	igt_assert(ret_tiling == buf->tiling);
	buf->swizzle_mode = ret_swizzle;
}

static unsigned long swizzle_bit(unsigned int bit, unsigned long offset)
{
	return (offset & (1ul << bit)) >> (bit - 6);
}

static unsigned long swizzle_addr(void *ptr, uint32_t swizzle)
{
	unsigned long addr = to_user_pointer(ptr);

	switch (swizzle) {
	case I915_BIT_6_SWIZZLE_NONE:
		return addr;
	case I915_BIT_6_SWIZZLE_9:
		return addr ^ swizzle_bit(9, addr);
	case I915_BIT_6_SWIZZLE_9_10:
		return addr ^ swizzle_bit(9, addr) ^ swizzle_bit(10, addr);
	case I915_BIT_6_SWIZZLE_9_11:
		return addr ^ swizzle_bit(9, addr) ^ swizzle_bit(11, addr);
	case I915_BIT_6_SWIZZLE_9_10_11:
		return (addr ^
			swizzle_bit(9, addr) ^
			swizzle_bit(10, addr) ^
			swizzle_bit(11, addr));

	case I915_BIT_6_SWIZZLE_UNKNOWN:
	case I915_BIT_6_SWIZZLE_9_17:
	case I915_BIT_6_SWIZZLE_9_10_17:
	default:
		igt_skip("physical swizzling mode impossible to handle in userspace\n");
		return addr;
	}
}

static void *linear_ptr(void *ptr,
			unsigned int x, unsigned int y,
			unsigned int stride, unsigned int cpp)
{
	int pos;

	pos = (stride/cpp * y + x) * cpp;

	return ptr + pos;
}

static void *x_ptr(void *ptr,
		   unsigned int x, unsigned int y,
		   unsigned int stride, unsigned int cpp,
		   const int tile_width, const int tile_height)
{
	const int tile_size = tile_width * tile_height;
	int offset_x, offset_y, pos;
	int tile_x, tile_y;

	x *= cpp;
	tile_x = x / tile_width;
	tile_y = y / tile_height;
	offset_x = tile_x * tile_size;
	offset_y = tile_y * stride * tile_height;

	pos = (offset_y + (y % tile_height * tile_width) +
	       offset_x + (x % tile_width));

	return ptr + pos;
}

static void *gen2_x_ptr(void *ptr,
			unsigned int x, unsigned int y,
			unsigned int stride, unsigned int cpp)
{
	return x_ptr(ptr, x, y, stride, cpp, 128, 16);
}

static void *gen3_x_ptr(void *ptr,
			unsigned int x, unsigned int y,
			unsigned int stride, unsigned int cpp)
{
	return x_ptr(ptr, x, y, stride, cpp, 512, 8);
}

static void *y_ptr(void *ptr,
		   unsigned int x, unsigned int y,
		   unsigned int stride, unsigned int cpp,
		   const int tile_width,
		   const int tile_height,
		   const int owords)
{
	const int tile_size = tile_width * tile_height;
	int offset_x, offset_y, pos;
	int shift_x, shift_y;
	int tile_x, tile_y;

	x *= cpp;
	tile_x = x / tile_width;
	tile_y = y / tile_height;
	offset_x = tile_x * tile_size;
	offset_y = tile_y * stride * tile_height;
	shift_x = x % owords + (x % tile_width) / owords * tile_width * cpp;
	shift_y = y % tile_height * owords;

	pos = offset_y + offset_x + shift_x + shift_y;

	return ptr + pos;
}

static void *gen2_y_ptr(void *ptr,
			unsigned int x, unsigned int y,
			unsigned int stride, unsigned int cpp)
{
	return y_ptr(ptr, x, y, stride, cpp, 128, 16, 8);
}

static void *i915_y_ptr(void *ptr,
			unsigned int x, unsigned int y,
			unsigned int stride, unsigned int cpp)
{
	return y_ptr(ptr, x, y, stride, cpp, 512, 8, 32);
}

static void *i945_y_ptr(void *ptr,
			unsigned int x, unsigned int y,
			unsigned int stride, unsigned int cpp)
{
	return y_ptr(ptr, x, y, stride, cpp, 128, 32, 16);
}

/*
 * (x,y) to memory location in tiled-4 surface
 *
 * coverted those divisions and multiplications to shifts and masks
 * in hope this wouldn't be so slow.
 */
static void *tile4_ptr(void *ptr,
			unsigned int x, unsigned int y,
			unsigned int stride, unsigned int cpp)
{
	const int tile_width = 128;
	const int tile_height = 32;
	const int subtile_size = 64;
	const int owords = 16;
	int base, _x, _y, subtile, tile_x, tile_y;
	int x_loc = x << __builtin_ctz(cpp);
	int pos;

	/* Pixel in tile via masks */
	tile_x = x_loc & (tile_width - 1);
	tile_y = y & (tile_height - 1);

	/* subtile in 4k tile */
	_x = tile_x >> __builtin_ctz(owords);
	_y = tile_y >> 2;

	/* tile-4 swizzle */
	subtile = ((_y >> 1) << 4) + ((_y & 1) << 2) + (_x & 3) + ((_x & 4) << 1);

	/* memory location */
	base = (y >> __builtin_ctz(tile_height)) *
		(stride << __builtin_ctz(tile_height)) +
		(((x_loc >> __builtin_ctz(tile_width)) << __builtin_ctz(4096)));

	pos = base + (subtile << __builtin_ctz(subtile_size)) +
		((tile_y & 3) << __builtin_ctz(owords)) +
		(tile_x & (owords - 1));
	igt_assert((pos & (cpp - 1)) == 0);

	return ptr + pos;
}


static void *yf_ptr(void *ptr,
		    unsigned int x, unsigned int y,
		    unsigned int stride, unsigned int cpp)
{
	const int tile_size = 4 * 1024;
	const int tile_width = 128;
	int row_size = stride / tile_width * tile_size;

	x *= cpp; /* convert to Byte offset */

	/*
	 * Within a 4k Yf tile, the byte swizzling pattern is
	 * msb......lsb
	 * xyxyxyyyxxxx
	 * The tiles themselves are laid out in row major order.
	 */
	return ptr +
		((x & 0xf) * 1) + /* 4x1 pixels(32bpp) = 16B */
		((y & 0x3) * 16) + /* 4x4 pixels = 64B */
		(((y & 0x4) >> 2) * 64) + /* 1x2 64B blocks */
		(((x & 0x10) >> 4) * 128) + /* 2x2 64B blocks = 256B block */
		(((y & 0x8) >> 3) * 256) + /* 2x1 256B blocks */
		(((x & 0x20) >> 5) * 512) + /* 2x2 256B blocks */
		(((y & 0x10) >> 4) * 1024) + /* 4x2 256 blocks */
		(((x & 0x40) >> 6) * 2048) + /* 4x4 256B blocks = 4k tile */
		(((x & ~0x7f) >> 7) * tile_size) + /* row of tiles */
		(((y & ~0x1f) >> 5) * row_size);
}

typedef void *(*tile_fn)(void *, unsigned int, unsigned int,
			unsigned int, unsigned int);
static tile_fn __get_tile_fn_ptr(int fd, int tiling)
{
	const struct intel_device_info *info =
		intel_get_device_info(intel_get_drm_devid(fd));
	tile_fn fn = NULL;

	switch (tiling) {
	case I915_TILING_NONE:
		fn = linear_ptr;
		break;
	case I915_TILING_X:
		if (info->graphics_ver == 2)
			fn = gen2_x_ptr;
		else
			fn = gen3_x_ptr;
		break;
	case I915_TILING_Y:
		if (info->graphics_ver == 2)
			fn = gen2_y_ptr;
		else if (info->is_grantsdale || info->is_alviso)
			fn = i915_y_ptr;
		else
			fn = i945_y_ptr;
		break;
	case I915_TILING_Yf:
		fn = yf_ptr;
		break;
	case I915_TILING_4:
		fn = tile4_ptr;
	case I915_TILING_Ys:
		/* To be implemented */
		break;
	}

	igt_require_f(fn, "Can't find tile function for tiling: %d\n", tiling);
	return fn;
}

static bool is_cache_coherent(int fd, uint32_t handle)
{
	return gem_get_caching(fd, handle) != I915_CACHING_NONE;
}

enum ccs_copy_direction {
	CCS_LINEAR_TO_BUF,
	CCS_BUF_TO_LINEAR,
};

static void __copy_ccs(struct buf_ops *bops, struct intel_buf *buf,
		       uint32_t *linear, enum ccs_copy_direction dir)
{
	uint64_t size, offset, ccs_size;
	unsigned int gen;
	void *map;

	if (!buf->compression || HAS_FLATCCS(intel_get_drm_devid(bops->fd)))
		return;

	gen = bops->intel_gen;
	offset = CCS_OFFSET(buf);
	ccs_size = CCS_SIZE(gen, buf);
	size = offset + ccs_size;

	if (gem_has_lmem(bops->fd)) {
		map = gem_mmap__device_coherent(bops->fd, buf->handle, 0, size,
						PROT_READ | PROT_WRITE);
	} else {
		map = __gem_mmap_offset__wc(bops->fd, buf->handle, 0, size,
					    PROT_READ | PROT_WRITE);
		if (!map)
			map = gem_mmap__wc(bops->fd, buf->handle, 0, size,
					   PROT_READ | PROT_WRITE);
	}

	switch (dir) {
	case CCS_LINEAR_TO_BUF:
		gem_set_domain(bops->fd, buf->handle,
			       I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC);
		igt_memcpy_from_wc(map + offset, (uint8_t *) linear + offset,
				   ccs_size);
	case CCS_BUF_TO_LINEAR:
		gem_set_domain(bops->fd, buf->handle, I915_GEM_DOMAIN_WC, 0);
		igt_memcpy_from_wc((uint8_t *) linear + offset, map + offset,
				   ccs_size);
	}

	munmap(map, size);
}

static void *mmap_write(int fd, const struct intel_buf *buf, bool *malloced)
{
	void *map = NULL;

	*malloced = false;

	if (buf->bops->driver == INTEL_DRIVER_XE)
		return xe_bo_map(fd, buf->handle, buf->surface[0].size);

	if (gem_has_lmem(fd)) {
		/*
		 * set/get_caching and set_domain are no longer supported on
		 * discrete, also the only mmap mode supportd is FIXED.
		 */
		map = gem_mmap_offset__fixed(fd, buf->handle, 0,
					     buf->surface[0].size,
					     PROT_READ | PROT_WRITE);
		igt_assert_eq(gem_wait(fd, buf->handle, 0), 0);
	}

	if (!map && is_cache_coherent(fd, buf->handle)) {
		map = __gem_mmap_offset__cpu(fd, buf->handle, 0, buf->surface[0].size,
					     PROT_READ | PROT_WRITE);
		if (!map)
			map = __gem_mmap__cpu(fd, buf->handle, 0, buf->surface[0].size,
					      PROT_READ | PROT_WRITE);

		if (map)
			gem_set_domain(fd, buf->handle,
				       I915_GEM_DOMAIN_CPU,
				       I915_GEM_DOMAIN_CPU);
	}

	if (!map && gem_mmap__has_wc(fd)) {
		map = __gem_mmap_offset__wc(fd, buf->handle, 0, buf->surface[0].size,
					    PROT_READ | PROT_WRITE);
		if (!map)
			map = gem_mmap__wc(fd, buf->handle, 0, buf->surface[0].size,
					   PROT_READ | PROT_WRITE);

		gem_set_domain(fd, buf->handle,
			       I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC);
	}

	if (!map) {
		map = malloc(buf->surface[0].size);
		igt_assert(map);
		*malloced = true;
	}

	return map;
}

static void munmap_write(void *map, int fd, const struct intel_buf *buf, bool malloced)
{
	if (malloced) {
		igt_assert(__gem_write(fd, buf->handle, 0, map, buf->surface[0].size) == 0);
		free(map);
	} else {
		munmap(map, buf->surface[0].size);
	}
}

static void *mmap_read(int fd, struct intel_buf *buf, bool *malloced)
{
	void *map = NULL;

	*malloced = false;

	if (buf->bops->driver == INTEL_DRIVER_XE)
		return xe_bo_map(fd, buf->handle, buf->surface[0].size);

	if (gem_has_lmem(fd)) {
		/*
		 * set/get_caching and set_domain are no longer supported on
		 * discrete, also the only supported mmap mode is FIXED.
		 */
		map = gem_mmap_offset__fixed(fd, buf->handle, 0,
					     buf->surface[0].size, PROT_READ);
		igt_assert_eq(gem_wait(fd, buf->handle, 0), 0);
	}

	if (!map && (gem_has_llc(fd) || is_cache_coherent(fd, buf->handle))) {
		map = __gem_mmap_offset__cpu(fd, buf->handle, 0,
					     buf->surface[0].size, PROT_READ);
		if (!map)
			map = __gem_mmap__cpu(fd, buf->handle, 0, buf->surface[0].size,
					      PROT_READ);

		if (map)
			gem_set_domain(fd, buf->handle, I915_GEM_DOMAIN_CPU, 0);
	}

	if (!map && gem_mmap__has_wc(fd)) {
		map = __gem_mmap_offset__wc(fd, buf->handle, 0, buf->surface[0].size,
					    PROT_READ);
		if (!map)
			map = gem_mmap__wc(fd, buf->handle, 0, buf->surface[0].size,
					   PROT_READ);

		gem_set_domain(fd, buf->handle, I915_GEM_DOMAIN_WC, 0);
	}

	if (!map) {
		map = malloc(buf->surface[0].size);
		igt_assert(map);
		*malloced = true;

		igt_assert(__gem_read(fd, buf->handle, 0, map, buf->surface[0].size) == 0);
	}

	return map;
}

static void munmap_read(void *map, int fd, const struct intel_buf *buf, bool malloced)
{
	if (malloced)
		free(map);
	else
		munmap(map, buf->surface[0].size);
}

static void __copy_linear_to(int fd, struct intel_buf *buf,
			     const uint32_t *linear,
			     int tiling, uint32_t swizzle)
{
	const tile_fn fn = __get_tile_fn_ptr(fd, tiling);
	int height = intel_buf_height(buf);
	int width = intel_buf_width(buf);
	bool malloced;
	void *map;

	map = mmap_write(fd, buf, &malloced);

	for (int y = 0; y < height; y++) {
		for (int x = 0; x < width; x++) {
			uint32_t *ptr = fn(map, x, y, buf->surface[0].stride, buf->bpp/8);

			if (swizzle)
				ptr = from_user_pointer(swizzle_addr(ptr,
								     swizzle));
			*ptr = linear[y * width + x];
		}
	}

	munmap_write(map, fd, buf, malloced);
}

static void copy_linear_to_none(struct buf_ops *bops, struct intel_buf *buf,
				uint32_t *linear)
{
	DEBUGFN();
	__copy_linear_to(bops->fd, buf, linear, I915_TILING_NONE, 0);
}

static void copy_linear_to_x(struct buf_ops *bops, struct intel_buf *buf,
			     uint32_t *linear)
{
	DEBUGFN();
	__copy_linear_to(bops->fd, buf, linear, I915_TILING_X, bops->swizzle_x);
}

static void copy_linear_to_y(struct buf_ops *bops, struct intel_buf *buf,
			     uint32_t *linear)
{
	DEBUGFN();
	__copy_linear_to(bops->fd, buf, linear, I915_TILING_Y, bops->swizzle_y);
}

static void copy_linear_to_yf(struct buf_ops *bops, struct intel_buf *buf,
			      uint32_t *linear)
{
	DEBUGFN();
	__copy_linear_to(bops->fd, buf, linear, I915_TILING_Yf, 0);
}

static void copy_linear_to_ys(struct buf_ops *bops, struct intel_buf *buf,
			      uint32_t *linear)
{
	DEBUGFN();
	__copy_linear_to(bops->fd, buf, linear, I915_TILING_Ys, 0);
}

static void copy_linear_to_tile4(struct buf_ops *bops, struct intel_buf *buf,
				 uint32_t *linear)
{
	DEBUGFN();
	__copy_linear_to(bops->fd, buf, linear, I915_TILING_4, 0);
}

static void __copy_to_linear(int fd, struct intel_buf *buf,
			     uint32_t *linear, int tiling, uint32_t swizzle)
{
	const tile_fn fn = __get_tile_fn_ptr(fd, tiling);
	int height = intel_buf_height(buf);
	int width = intel_buf_width(buf);
	bool malloced;
	void *map;

	map = mmap_write(fd, buf, &malloced);

	for (int y = 0; y < height; y++) {
		for (int x = 0; x < width; x++) {
			uint32_t *ptr = fn(map, x, y, buf->surface[0].stride, buf->bpp/8);

			if (swizzle)
				ptr = from_user_pointer(swizzle_addr(ptr,
								     swizzle));
			linear[y * width + x] = *ptr;
		}
	}

	munmap_write(map, fd, buf, malloced);
}

static void copy_none_to_linear(struct buf_ops *bops, struct intel_buf *buf,
				uint32_t *linear)
{
	DEBUGFN();
	__copy_to_linear(bops->fd, buf, linear, I915_TILING_NONE, 0);
}

static void copy_x_to_linear(struct buf_ops *bops, struct intel_buf *buf,
			     uint32_t *linear)
{
	DEBUGFN();
	__copy_to_linear(bops->fd, buf, linear, I915_TILING_X, bops->swizzle_x);
}

static void copy_y_to_linear(struct buf_ops *bops, struct intel_buf *buf,
			     uint32_t *linear)
{
	DEBUGFN();
	__copy_to_linear(bops->fd, buf, linear, I915_TILING_Y, bops->swizzle_y);
}

static void copy_yf_to_linear(struct buf_ops *bops, struct intel_buf *buf,
			      uint32_t *linear)
{
	DEBUGFN();
	__copy_to_linear(bops->fd, buf, linear, I915_TILING_Yf, 0);
}

static void copy_ys_to_linear(struct buf_ops *bops, struct intel_buf *buf,
			      uint32_t *linear)
{
	DEBUGFN();
	__copy_to_linear(bops->fd, buf, linear, I915_TILING_Ys, 0);
}

static void copy_tile4_to_linear(struct buf_ops *bops, struct intel_buf *buf,
				 uint32_t *linear)
{
	DEBUGFN();
	__copy_to_linear(bops->fd, buf, linear, I915_TILING_4, 0);
}

static void copy_linear_to_gtt(struct buf_ops *bops, struct intel_buf *buf,
			       uint32_t *linear)
{
	void *map;

	DEBUGFN();

	map = gem_mmap__gtt(bops->fd, buf->handle,
			    buf->surface[0].size, PROT_READ | PROT_WRITE);

	gem_set_domain(bops->fd, buf->handle,
		       I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);

	memcpy(map, linear, buf->surface[0].size);

	munmap(map, buf->surface[0].size);
}

static void copy_gtt_to_linear(struct buf_ops *bops, struct intel_buf *buf,
			       uint32_t *linear)
{
	void *map;

	DEBUGFN();

	map = gem_mmap__gtt(bops->fd, buf->handle,
			    buf->surface[0].size, PROT_READ);

	gem_set_domain(bops->fd, buf->handle,
		       I915_GEM_DOMAIN_GTT, 0);

	igt_memcpy_from_wc(linear, map, buf->surface[0].size);

	munmap(map, buf->surface[0].size);
}

static void copy_linear_to_wc(struct buf_ops *bops, struct intel_buf *buf,
			      uint32_t *linear)
{
	bool malloced;
	void *map;

	DEBUGFN();

	map = mmap_write(bops->fd, buf, &malloced);
	memcpy(map, linear, buf->surface[0].size);
	munmap_write(map, bops->fd, buf, malloced);
}

static void copy_wc_to_linear(struct buf_ops *bops, struct intel_buf *buf,
			      uint32_t *linear)
{
	bool malloced;
	void *map;

	DEBUGFN();

	map = mmap_read(bops->fd, buf, &malloced);
	igt_memcpy_from_wc(linear, map, buf->surface[0].size);
	munmap_read(map, bops->fd, buf, malloced);
}

void intel_buf_to_linear(struct buf_ops *bops, struct intel_buf *buf,
			 uint32_t *linear)
{
	igt_assert(bops);

	switch (buf->tiling) {
	case I915_TILING_NONE:
		igt_assert(bops->to_linear);
		bops->to_linear(bops, buf, linear);
		break;
	case I915_TILING_X:
		igt_assert(bops->x_to_linear);
		bops->x_to_linear(bops, buf, linear);
		break;
	case I915_TILING_Y:
		igt_assert(bops->y_to_linear);
		bops->y_to_linear(bops, buf, linear);
		break;
	case I915_TILING_Yf:
		igt_assert(bops->yf_to_linear);
		bops->yf_to_linear(bops, buf, linear);
		break;
	case I915_TILING_Ys:
		igt_assert(bops->ys_to_linear);
		bops->ys_to_linear(bops, buf, linear);
		break;
	case I915_TILING_4:
		igt_assert(bops->tile4_to_linear);
		bops->tile4_to_linear(bops, buf, linear);
		break;
	}

	if (buf->compression)
		__copy_ccs(bops, buf, linear, CCS_BUF_TO_LINEAR);
}

void linear_to_intel_buf(struct buf_ops *bops, struct intel_buf *buf,
			 uint32_t *linear)
{
	igt_assert(bops);

	switch (buf->tiling) {
	case I915_TILING_NONE:
		igt_assert(bops->linear_to);
		bops->linear_to(bops, buf, linear);
		break;
	case I915_TILING_X:
		igt_assert(bops->linear_to_x);
		bops->linear_to_x(bops, buf, linear);
		break;
	case I915_TILING_Y:
		igt_assert(bops->linear_to_y);
		bops->linear_to_y(bops, buf, linear);
		break;
	case I915_TILING_Yf:
		igt_assert(bops->linear_to_yf);
		bops->linear_to_yf(bops, buf, linear);
		break;
	case I915_TILING_Ys:
		igt_assert(bops->linear_to_ys);
		bops->linear_to_ys(bops, buf, linear);
		break;
	case I915_TILING_4:
		igt_assert(bops->linear_to_tile4);
		bops->linear_to_tile4(bops, buf, linear);
		break;
	}

	if (buf->compression)
		__copy_ccs(bops, buf, linear, CCS_LINEAR_TO_BUF);
}

static uint32_t __get_min_stride(uint32_t width, uint32_t bpp, int tiling)
{
	switch (tiling) {
	case I915_TILING_NONE:
		return width * bpp / 8;
	case I915_TILING_X:
		return ALIGN(width * bpp / 8, 512);
	case I915_TILING_64:
		if (bpp == 8)
			return ALIGN(width, 256);
		else if (bpp == 16 || bpp == 32)
			return ALIGN(width * bpp / 8, 512);
		return ALIGN(width * bpp / 8, 1024);

	default:
		return ALIGN(width * bpp / 8, 128);
	}
}

static uint32_t __get_aligned_height(uint32_t height, uint32_t bpp, int tiling)
{
	switch (tiling) {
	case I915_TILING_NONE:
		return height;
	case I915_TILING_X:
		return ALIGN(height, 8);
	case I915_TILING_64:
		if (bpp == 8)
			return ALIGN(height, 256);
		else if (bpp == 16 || bpp == 32)
			return ALIGN(height, 128);
		return ALIGN(height, 64);
	default:
		return ALIGN(height, 32);
	}
}

static void __intel_buf_init(struct buf_ops *bops,
			     uint32_t handle,
			     struct intel_buf *buf,
			     int width, int height, int bpp, int alignment,
			     uint32_t req_tiling, uint32_t compression,
			     uint64_t bo_size, int bo_stride,
			     uint64_t region, uint8_t pat_index,
			     uint8_t mocs_index)
{
	uint32_t tiling = req_tiling;
	uint64_t size;
	int tile_width, aligned_height;

	igt_assert(bops);
	igt_assert(buf);
	igt_assert(width > 0 && height > 0);
	igt_assert(bpp == 8 || bpp == 16 || bpp == 32 || bpp == 64);
	igt_assert(alignment >= 0);

	memset(buf, 0, sizeof(*buf));

	buf->bops = bops;
	buf->width = width;
	buf->height = height;
	buf->tiling = tiling;
	buf->bpp = bpp;
	buf->compression = compression;
	buf->addr.offset = INTEL_BUF_INVALID_ADDRESS;
	buf->pat_index = pat_index;
	if (mocs_index == DEFAULT_MOCS_INDEX)
		mocs_index = intel_get_uc_mocs_index(bops->fd);
	buf->mocs_index = mocs_index;
	IGT_INIT_LIST_HEAD(&buf->link);

	tile_width = __get_min_stride(width, bpp, tiling);
	aligned_height = __get_aligned_height(height, bpp, tiling);

	if (bo_stride)
		buf->surface[0].stride = bo_stride;
	else
		buf->surface[0].stride = tile_width;

	size = buf->surface[0].size = buf->surface[0].stride * aligned_height;

	if (compression && !HAS_FLATCCS(buf_ops_get_devid(bops))) {
		int aux_width, aux_height;

		igt_require(bops->intel_gen >= 9);
		igt_assert(req_tiling == I915_TILING_Y ||
			   req_tiling == I915_TILING_Yf ||
			   req_tiling == I915_TILING_4);
		/*
		 * On GEN12+ we align the main surface to 4 * 4 main surface
		 * tiles, which is 64kB. These 16 tiles are mapped by 4 AUX
		 * CCS units, that is 4 * 64 bytes. These 4 CCS units are in
		 * turn mapped by one L1 AUX page table entry.
		 */
		aux_width = intel_buf_ccs_width(bops->intel_gen, buf);
		aux_height = intel_buf_ccs_height(bops->intel_gen, buf);

		buf->ccs[0].offset = buf->surface[0].stride * ALIGN(height, 32);
		buf->ccs[0].stride = aux_width;
		size = buf->ccs[0].offset + aux_width * aux_height;
	}

	/* Store buffer size to avoid mistakes in calculating it again */
	buf->size = size;
	buf->handle = handle;

	if (bops->driver == INTEL_DRIVER_XE)
		igt_assert_f(region != -1, "Xe requires region awareness, "
					   "use api which passes valid region\n");
	buf->region = region;

	if (!handle) {
		if (!bo_size)
			bo_size = size;

		if (bops->driver == INTEL_DRIVER_I915) {
			if (__gem_create_in_memory_regions(bops->fd, &buf->handle, &bo_size, region))
				igt_assert_eq(__gem_create(bops->fd, &bo_size, &buf->handle), 0);
		} else {
			uint16_t cpu_caching = __xe_default_cpu_caching(bops->fd, region, 0);

			if (intel_gen(bops->devid) >= 20 && compression)
				cpu_caching = DRM_XE_GEM_CPU_CACHING_WC;

			bo_size = ALIGN(bo_size, xe_get_default_alignment(bops->fd));
			buf->handle = xe_bo_create_caching(bops->fd, 0, bo_size, region,
							   DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM,
							   cpu_caching);
		}
	}

	igt_assert(bo_size >= size);

	/* Store gem bo size */
	buf->bo_size = bo_size;

	if (bops->driver == INTEL_DRIVER_I915)
		set_hw_tiled(bops, buf);
}

/**
 * intel_buf_init
 * @bops: pointer to buf_ops
 * @buf: pointer to intel_buf structure to be filled
 * @width: surface width
 * @height: surface height
 * @bpp: bits-per-pixel (8 / 16 / 32 / 64)
 * @alignment: alignment of the stride for linear surfaces
 * @tiling: surface tiling
 * @compression: surface compression type
 *
 * Function creates new BO within intel_buf structure and fills all
 * structure fields. Takes bo handle ownership.
 *
 * Note. For X / Y if GPU supports fences HW tiling is configured.
 */
void intel_buf_init(struct buf_ops *bops,
		    struct intel_buf *buf,
		    int width, int height, int bpp, int alignment,
		    uint32_t tiling, uint32_t compression)
{
	uint64_t region;
	uint8_t pat_index = DEFAULT_PAT_INDEX;

	if (compression && intel_gen(bops->devid) >= 20)
		pat_index = intel_get_pat_idx_uc_comp(bops->fd);

	region = bops->driver == INTEL_DRIVER_I915 ? I915_SYSTEM_MEMORY :
						     system_memory(bops->fd);
	__intel_buf_init(bops, 0, buf, width, height, bpp, alignment,
			 tiling, compression, 0, 0, region,
			 pat_index,
			 DEFAULT_MOCS_INDEX);

	intel_buf_set_ownership(buf, true);
}

/**
 * intel_buf_init_in_region
 *
 * Same as intel_buf_init with the additional region argument
 */
void intel_buf_init_in_region(struct buf_ops *bops,
			      struct intel_buf *buf,
			      int width, int height, int bpp, int alignment,
			      uint32_t tiling, uint32_t compression,
			      uint64_t region)
{
	uint8_t pat_index = DEFAULT_PAT_INDEX;

	if (compression && intel_gen(bops->devid) >= 20)
		pat_index = intel_get_pat_idx_uc_comp(bops->fd);

	__intel_buf_init(bops, 0, buf, width, height, bpp, alignment,
			 tiling, compression, 0, 0, region,
			 pat_index,
			 DEFAULT_MOCS_INDEX);

	intel_buf_set_ownership(buf, true);
}

/**
 * intel_buf_close
 * @bops: pointer to buf_ops
 * @buf: pointer to intel_buf structure
 *
 * Function closes gem BO inside intel_buf if bo is owned by intel_buf.
 * For handle passed from the caller intel_buf doesn't take ownership and
 * doesn't close it in close()/destroy() paths. When intel_buf was previously
 * added to intel_bb (intel_bb_add_intel_buf() call) it is tracked there and
 * must be removed from its internal structures.
 */
void intel_buf_close(struct buf_ops *bops, struct intel_buf *buf)
{
	igt_assert(bops);
	igt_assert(buf);

	/* If buf is tracked by some intel_bb ensure it will be removed there */
	if (buf->ibb) {
		intel_bb_remove_intel_buf(buf->ibb, buf);
		buf->addr.offset = INTEL_BUF_INVALID_ADDRESS;
		buf->ibb = NULL;
		IGT_INIT_LIST_HEAD(&buf->link);
	}

	if (buf->is_owner)
		gem_close(bops->fd, buf->handle);
}

/**
 * intel_buf_init_using_handle_and_size
 * @bops: pointer to buf_ops
 * @handle: BO handle created by the caller
 * @buf: pointer to intel_buf structure to be filled
 * @width: surface width
 * @height: surface height
 * @bpp: bits-per-pixel (8 / 16 / 32 / 64)
 * @alignment: alignment of the stride for linear surfaces
 * @tiling: surface tiling
 * @compression: surface compression type
 * @size: real bo size
 *
 * Function configures BO handle within intel_buf structure passed by the caller
 * (with all its metadata - width, height, ...). Useful if BO was created
 * outside.
 *
 * Note: intel_buf_close() can be used because intel_buf is aware it is not
 * buffer owner so it won't close it underneath.
 */
void intel_buf_init_using_handle_and_size(struct buf_ops *bops,
					  uint32_t handle,
					  struct intel_buf *buf,
					  int width, int height, int bpp, int alignment,
					  uint32_t req_tiling, uint32_t compression,
					  uint64_t size)
{
	uint8_t pat_index = DEFAULT_PAT_INDEX;

	igt_assert(handle);
	igt_assert(size);

	if (compression && intel_gen(bops->devid) >= 20)
		pat_index = intel_get_pat_idx_uc_comp(bops->fd);

	__intel_buf_init(bops, handle, buf, width, height, bpp, alignment,
			 req_tiling, compression, size, 0, -1, pat_index,
			 DEFAULT_MOCS_INDEX);
}

/**
 * intel_buf_init_full
 * @bops: pointer to buf_ops
 * @handle: BO handle created by the caller
 * @buf: pointer to intel_buf structure to be filled
 * @width: surface width
 * @height: surface height
 * @bpp: bits-per-pixel (8 / 16 / 32 / 64)
 * @alignment: alignment of the stride for linear surfaces
 * @req_tiling: surface tiling
 * @compression: surface compression type
 * @size: real bo size
 * @stride: bo stride
 * @region: region
 * @pat_index: pat_index to use for the binding (only used on xe)
 * @pat_index: mocs_index to use for operations using this intel_buf, like render
 * copy.
 *
 * Function configures BO handle within intel_buf structure passed by the caller
 * (with all its metadata - width, height, ...). Useful if BO was created
 * outside. Allows passing real size which caller is aware of.
 *
 * Note: intel_buf_close() can be used because intel_buf is aware it is not
 * buffer owner so it won't close it underneath.
 */
void intel_buf_init_full(struct buf_ops *bops,
			 uint32_t handle,
			 struct intel_buf *buf,
			 int width, int height,
			 int bpp, int alignment,
			 uint32_t req_tiling,
			 uint32_t compression,
			 uint64_t size,
			 int stride,
			 uint64_t region,
			 uint8_t pat_index,
			 uint8_t mocs_index)
{
	__intel_buf_init(bops, handle, buf, width, height, bpp, alignment,
			 req_tiling, compression, size, stride, region,
			 pat_index, mocs_index);
}

/**
 * intel_buf_create
 * @bops: pointer to buf_ops
 * @width: surface width
 * @height: surface height
 * @bpp: bits-per-pixel (8 / 16 / 32 / 64)
 * @alignment: alignment of the stride for linear surfaces
 * @tiling: surface tiling
 * @compression: surface compression type
 *
 * Function creates intel_buf with created BO handle. Takes ownership of the
 * buffer.
 */
struct intel_buf *intel_buf_create(struct buf_ops *bops,
				   int width, int height,
				   int bpp, int alignment,
				   uint32_t req_tiling, uint32_t compression)
{
	struct intel_buf *buf;

	igt_assert(bops);

	buf = calloc(1, sizeof(*buf));
	igt_assert(buf);

	intel_buf_init(bops, buf, width, height, bpp, alignment,
		       req_tiling, compression);

	return buf;
}

/**
 * intel_buf_create_using_handle_and_size
 * @bops: pointer to buf_ops
 * @handle: BO handle created by the caller
 * @width: surface width
 * @height: surface height
 * @bpp: bits-per-pixel (8 / 16 / 32 / 64)
 * @alignment: alignment of the stride for linear surfaces
 * @tiling: surface tiling
 * @compression: surface compression type
 * @size: real bo size
 *
 * Function creates intel_buf with passed BO handle from the caller. Doesn't
 * take ownership of the buffer. close()/destroy() paths doesn't close
 * passed handle unless buffer will take ownership using set_ownership().
 */
struct intel_buf *intel_buf_create_using_handle_and_size(struct buf_ops *bops,
							 uint32_t handle,
							 int width, int height,
							 int bpp, int alignment,
							 uint32_t req_tiling,
							 uint32_t compression,
							 uint64_t size)
{
	igt_assert(handle);
	igt_assert(size);
	return intel_buf_create_full(bops, handle, width, height, bpp, alignment,
				     req_tiling, compression, size, 0, -1,
				     DEFAULT_PAT_INDEX, DEFAULT_MOCS_INDEX);
}

struct intel_buf *intel_buf_create_full(struct buf_ops *bops,
					uint32_t handle,
					int width, int height,
					int bpp, int alignment,
					uint32_t req_tiling,
					uint32_t compression,
					uint64_t size,
					int stride,
					uint64_t region,
					uint8_t pat_index,
					uint8_t mocs_index)
{
	struct intel_buf *buf;

	igt_assert(bops);

	buf = calloc(1, sizeof(*buf));
	igt_assert(buf);

	__intel_buf_init(bops, handle, buf, width, height, bpp, alignment,
			 req_tiling, compression, size, stride, region,
			 pat_index, mocs_index);

	return buf;
}

/**
 * intel_buf_destroy
 * @buf: intel_buf
 *
 * Function frees intel_buf memory. It closes bo handle if intel_buf has
 * buffer ownership.
 */
void intel_buf_destroy(struct intel_buf *buf)
{
	igt_assert(buf);
	igt_assert(buf->ptr == NULL);

	intel_buf_close(buf->bops, buf);

	free(buf);
}

void *intel_buf_cpu_map(struct intel_buf *buf, bool write)
{
	int fd = buf_ops_get_fd(buf->bops);

	igt_assert(buf);
	igt_assert(buf->ptr == NULL); /* already mapped */

	buf->cpu_write = write;

	if (is_xe_device(fd)) {
		buf->ptr = xe_bo_map(fd, buf->handle, buf->bo_size);
	} else {
		buf->ptr = gem_mmap__cpu_coherent(fd, buf->handle, 0,
						  buf->bo_size,
						  write ? PROT_WRITE : PROT_READ);

		gem_set_domain(fd, buf->handle,
			       I915_GEM_DOMAIN_CPU,
			       write ? I915_GEM_DOMAIN_CPU : 0);
	}

	return buf->ptr;
}

void *intel_buf_device_map(struct intel_buf *buf, bool write)
{
	int fd = buf_ops_get_fd(buf->bops);

	igt_assert(buf);
	igt_assert(buf->ptr == NULL); /* already mapped */

	if (is_xe_device(fd)) {
		buf->ptr = xe_bo_map(fd, buf->handle, buf->bo_size);
	} else {
		buf->ptr = gem_mmap__device_coherent(fd, buf->handle, 0,
						     buf->bo_size,
						     write ? PROT_WRITE : PROT_READ);

		gem_set_domain(fd, buf->handle,
			       I915_GEM_DOMAIN_WC,
			       write ? I915_GEM_DOMAIN_WC : 0);
	}

	return buf->ptr;
}

void intel_buf_unmap(struct intel_buf *buf)
{
	igt_assert(buf);
	igt_assert(buf->ptr);

	munmap(buf->ptr, buf->bo_size);
	buf->ptr = NULL;
}

void intel_buf_flush_and_unmap(struct intel_buf *buf)
{
	igt_assert(buf);
	igt_assert(buf->ptr);

	if (buf->cpu_write)
		gem_sw_finish(buf_ops_get_fd(buf->bops), buf->handle);

	intel_buf_unmap(buf);
}

void intel_buf_print(const struct intel_buf *buf)
{
	igt_info("[name: %s]\n", buf->name);
	igt_info("[%u]: w: %u, h: %u, stride: %u, size: %" PRIx64
		 ", buf-size: %" PRIx64 ", bo-size: %" PRIx64
		 ", bpp: %u, tiling: %u, compress: %u\n",
		 buf->handle, intel_buf_width(buf), intel_buf_height(buf),
		 buf->surface[0].stride, buf->surface[0].size,
		 intel_buf_size(buf), intel_buf_bo_size(buf), buf->bpp,
		 buf->tiling, buf->compression);
	igt_info(" ccs <offset: %u, stride: %u, w: %u, h: %u> cc <offset: %u>\n",
		 buf->ccs[0].offset,
		 intel_buf_ccs_width(buf->bops->intel_gen, buf),
		 intel_buf_ccs_height(buf->bops->intel_gen, buf),
		 buf->ccs[0].stride, buf->cc.offset);
	igt_info(" addr <offset: %p, ctx: %u>\n",
		 from_user_pointer(buf->addr.offset), buf->addr.ctx);
}

void intel_buf_dump(const struct intel_buf *buf, const char *filename)
{
	int i915 = buf_ops_get_fd(buf->bops);
	uint64_t size = intel_buf_size(buf);
	FILE *out;
	void *ptr;

	ptr = gem_mmap__device_coherent(i915, buf->handle, 0, size, PROT_READ);
	out = fopen(filename, "wb");
	igt_assert(out);
	fwrite(ptr, size, 1, out);
	fclose(out);
	munmap(ptr, size);
}

const char *intel_buf_set_name(struct intel_buf *buf, const char *name)
{
	return strncpy(buf->name, name, INTEL_BUF_NAME_MAXSIZE);
}

static void __intel_buf_write_to_png(struct buf_ops *bops,
				     struct intel_buf *buf,
				     const char *filename,
				     bool write_ccs)
{
	cairo_surface_t *surface;
	cairo_status_t ret;
	void *linear;
	int format, width, height, stride, offset;
	unsigned int gen = bops->intel_gen;

	igt_assert_eq(posix_memalign(&linear, 16, intel_buf_size(buf)), 0);

	format = write_ccs ? CAIRO_FORMAT_A8 : CAIRO_FORMAT_RGB24;
	width = write_ccs ? intel_buf_ccs_width(gen, buf) : intel_buf_width(buf);
	height = write_ccs ? intel_buf_ccs_height(gen, buf) : intel_buf_height(buf);
	stride = write_ccs ? buf->ccs[0].stride : buf->surface[0].stride;
	offset = write_ccs ? buf->ccs[0].offset : 0;

	intel_buf_to_linear(bops, buf, linear);

	surface = cairo_image_surface_create_for_data((uint8_t *) linear + offset,
						      format, width, height,
						      stride);
	ret = cairo_surface_write_to_png(surface, filename);
	igt_assert(ret == CAIRO_STATUS_SUCCESS);
	cairo_surface_destroy(surface);

	free(linear);
}

void intel_buf_write_to_png(struct intel_buf *buf, const char *filename)
{
	__intel_buf_write_to_png(buf->bops, buf, filename, false);
}

void intel_buf_write_aux_to_png(struct intel_buf *buf, const char *filename)
{
	igt_assert(buf->compression);

	__intel_buf_write_to_png(buf->bops, buf, filename, true);
}
static void __intel_buf_raw_write_to_png(struct buf_ops *bops,
					 struct intel_buf *buf,
					 const char *filename)
{
	cairo_surface_t *surface;
	cairo_status_t ret;
	uint8_t *linear;
	int format, width, height, stride;

	format = CAIRO_FORMAT_RGB24;
	width = buf->surface[0].stride / 4;
	height = __get_aligned_height(intel_buf_height(buf),
				      buf->bpp, buf->tiling);
	stride = buf->surface[0].stride;

	if (is_xe_device(bops->fd))
		linear = xe_bo_map(bops->fd, buf->handle, buf->bo_size);
	else
		linear = gem_mmap__device_coherent(bops->fd, buf->handle,
						   0, buf->bo_size, PROT_READ);
	surface = cairo_image_surface_create_for_data(linear,
						      format, width, height,
						      stride);
	ret = cairo_surface_write_to_png(surface, filename);
	igt_assert(ret == CAIRO_STATUS_SUCCESS);
	cairo_surface_destroy(surface);

	munmap(linear, buf->bo_size);
}

__attribute__((format(printf, 2, 3)))
void intel_buf_raw_write_to_png(struct intel_buf *buf, const char *namefmt, ...)
{
	char *filename;
	int ret;
	va_list ap;

	va_start(ap, namefmt);
	ret = vasprintf(&filename, namefmt, ap);
	igt_assert(ret >= 0);
	va_end(ap);

	__intel_buf_raw_write_to_png(buf->bops, buf, filename);

	free(filename);
}

static void *alloc_aligned(uint64_t size)
{
	void *p;

	igt_assert_eq(posix_memalign(&p, 16, size), 0);

	return p;
}

void intel_buf_draw_pattern(struct buf_ops *bops, struct intel_buf *buf,
			    int x, int y, int w, int h,
			    int cx, int cy, int cw, int ch,
			    bool use_alternate_colors)
{
	cairo_surface_t *surface;
	cairo_pattern_t *pat;
	cairo_t *cr;
	void *linear;

	linear = alloc_aligned(buf->surface[0].size);

	surface = cairo_image_surface_create_for_data(linear,
						      CAIRO_FORMAT_RGB24,
						      intel_buf_width(buf),
						      intel_buf_height(buf),
						      buf->surface[0].stride);

	cr = cairo_create(surface);

	cairo_rectangle(cr, cx, cy, cw, ch);
	cairo_clip(cr);

	pat = cairo_pattern_create_mesh();
	cairo_mesh_pattern_begin_patch(pat);
	cairo_mesh_pattern_move_to(pat, x,   y);
	cairo_mesh_pattern_line_to(pat, x+w, y);
	cairo_mesh_pattern_line_to(pat, x+w, y+h);
	cairo_mesh_pattern_line_to(pat, x,   y+h);
	if (use_alternate_colors) {
		cairo_mesh_pattern_set_corner_color_rgb(pat, 0, 0.0, 1.0, 1.0);
		cairo_mesh_pattern_set_corner_color_rgb(pat, 1, 1.0, 0.0, 1.0);
		cairo_mesh_pattern_set_corner_color_rgb(pat, 2, 1.0, 1.0, 0.0);
		cairo_mesh_pattern_set_corner_color_rgb(pat, 3, 0.0, 0.0, 0.0);
	} else {
		cairo_mesh_pattern_set_corner_color_rgb(pat, 0, 1.0, 0.0, 0.0);
		cairo_mesh_pattern_set_corner_color_rgb(pat, 1, 0.0, 1.0, 0.0);
		cairo_mesh_pattern_set_corner_color_rgb(pat, 2, 0.0, 0.0, 1.0);
		cairo_mesh_pattern_set_corner_color_rgb(pat, 3, 1.0, 1.0, 1.0);
	}
	cairo_mesh_pattern_end_patch(pat);

	cairo_rectangle(cr, x, y, w, h);
	cairo_set_source(cr, pat);
	cairo_fill(cr);
	cairo_pattern_destroy(pat);

	cairo_destroy(cr);

	cairo_surface_destroy(surface);

	linear_to_intel_buf(bops, buf, linear);

	free(linear);
}

#define DEFAULT_BUFOPS(__gen_start, __gen_end) \
	.gen_start          = __gen_start, \
	.gen_end            = __gen_end, \
	.supported_hw_tiles = TILE_X | TILE_Y, \
	.linear_to          = copy_linear_to_wc, \
	.linear_to_x        = copy_linear_to_gtt, \
	.linear_to_y        = copy_linear_to_gtt, \
	.linear_to_yf       = copy_linear_to_yf, \
	.linear_to_ys       = copy_linear_to_ys, \
	.linear_to_tile4    = copy_linear_to_tile4, \
	.to_linear          = copy_wc_to_linear, \
	.x_to_linear        = copy_gtt_to_linear, \
	.y_to_linear        = copy_gtt_to_linear, \
	.yf_to_linear       = copy_yf_to_linear, \
	.ys_to_linear       = copy_ys_to_linear, \
	.tile4_to_linear    = copy_tile4_to_linear

static const struct buf_ops buf_ops_arr[] = {
	{
		DEFAULT_BUFOPS(2, 8),
		.supported_tiles    = TILE_NONE | TILE_X | TILE_Y,
	},

	{
		DEFAULT_BUFOPS(9, 11),
		.supported_tiles    = TILE_NONE | TILE_X | TILE_Y | TILE_Yf,
	},

	{
		DEFAULT_BUFOPS(12, ~0U),
		.supported_tiles   = TILE_NONE | TILE_X | TILE_Y | TILE_Yf | TILE_Ys | TILE_4,
	},
};

static bool probe_hw_tiling(struct buf_ops *bops, uint32_t tiling,
			    bool *swizzling_supported)
{
	uint64_t size = 256 * 256;
	uint32_t handle, buf_tiling, buf_swizzle, phys_swizzle;
	uint32_t stride;
	int ret;
	bool is_set = false;

	stride = get_stride(bops->devid, tiling);
	handle = gem_create(bops->fd, size);

	/* Single shot, if no fences are available we fail immediately */
	ret = __set_tiling(bops->fd, handle, tiling, stride, NULL, NULL);
	if (ret)
		goto end;

	is_set = __get_tiling(bops->fd, handle, &buf_tiling, &buf_swizzle,
			      &phys_swizzle);
	if (is_set) {
		if (tiling == I915_TILING_X)
			bops->swizzle_x = buf_swizzle;
		else if (tiling == I915_TILING_Y)
			bops->swizzle_y = buf_swizzle;

		*swizzling_supported = buf_swizzle == phys_swizzle;
	}
end:
	gem_close(bops->fd, handle);

	return is_set;
}

/*
 * Simple idempotency test between HW -> SW and SW -> HW BO.
 */
static void idempotency_selftest(struct buf_ops *bops, uint32_t tiling)
{
	struct intel_buf buf;
	uint8_t *linear_in, *linear_out, *map;
	int i;
	const int width = 512;
	const int height = 512;
	const int bpp = 32;
	const int size = width * height * bpp / 8;
	bool software_tiling = false;

	if (!is_hw_tiling_supported(bops, tiling))
		return;

	linear_in = alloc_aligned(size);
	linear_out = alloc_aligned(size);

	for (i = 0; i < size; i++)
		linear_in[i] = i % 253; /* prime chosen intentionally */

	do {
		igt_debug("Checking idempotency, SW: %s, HW: %s, tiling: %s\n",
			  bool_str(software_tiling),
			  bool_str(!software_tiling),
			  tiling_str(tiling));
		intel_buf_init(bops, &buf, width, height, bpp, 0, tiling, false);
		buf_ops_set_software_tiling(bops, tiling, software_tiling);

		linear_to_intel_buf(bops, &buf, (uint32_t *) linear_in);
		map = __gem_mmap_offset__cpu(bops->fd, buf.handle, 0,
					     buf.surface[0].size, PROT_READ);
		if (!map)
			map = gem_mmap__cpu(bops->fd, buf.handle, 0,
					    buf.surface[0].size, PROT_READ);
		gem_set_domain(bops->fd, buf.handle, I915_GEM_DOMAIN_CPU, 0);
		igt_assert(memcmp(linear_in, map, size));
		munmap(map, size);

		buf_ops_set_software_tiling(bops, tiling, !software_tiling);
		intel_buf_to_linear(bops, &buf, (uint32_t *) linear_out);
		igt_assert(!memcmp(linear_in, linear_out, size));

		intel_buf_close(bops, &buf);

		software_tiling = !software_tiling;
	} while (software_tiling);

	igt_debug("Idempotency for %s tiling OK\n", tiling_str(tiling));
	buf_ops_set_software_tiling(bops, tiling, false);
}

uint64_t intel_buf_size(const struct intel_buf *buf)
{
	return buf->size;
}

uint64_t intel_buf_bo_size(const struct intel_buf *buf)
{
	return buf->bo_size;
}

static struct buf_ops *__buf_ops_create(int fd, bool check_idempotency)
{
	struct buf_ops *bops = calloc(1, sizeof(*bops));
	unsigned int generation;
	uint32_t devid;

	igt_assert(bops);

	devid = intel_get_drm_devid(fd);
	generation = intel_gen(devid);

	/* Predefined settings: see intel_device_info? */
	for (int i = 0; i < ARRAY_SIZE(buf_ops_arr); i++) {
		if (generation >= buf_ops_arr[i].gen_start &&
		    generation <= buf_ops_arr[i].gen_end) {
			memcpy(bops, &buf_ops_arr[i], sizeof(*bops));
			break;
		}
	}

	bops->fd = fd;
	bops->intel_gen = generation;
	bops->devid = devid;
	bops->driver = is_i915_device(fd) ? INTEL_DRIVER_I915 :
					    is_xe_device(fd) ? INTEL_DRIVER_XE : 0;
	igt_assert(bops->driver);
	igt_debug("generation: %d, supported tiles: 0x%02x, driver: %s\n",
		  bops->intel_gen, bops->supported_tiles,
		  bops->driver == INTEL_DRIVER_I915 ? "i915" : "xe");

	if (bops->driver == INTEL_DRIVER_XE) {
		bops->linear_to = copy_linear_to_none;
		bops->to_linear = copy_none_to_linear;
		bops->linear_to_x = copy_linear_to_x;
		bops->x_to_linear = copy_x_to_linear;
		bops->linear_to_y = copy_linear_to_y;
		bops->y_to_linear = copy_y_to_linear;

		return bops;
	}

	/* Let's probe X and Y hw tiling support */
	if (is_hw_tiling_supported(bops, I915_TILING_X)) {
		bool swizzling_supported;
		bool supported = probe_hw_tiling(bops, I915_TILING_X,
						 &swizzling_supported);

		if (!swizzling_supported && bops->intel_gen < 12) {
			igt_debug("Swizzling for X is not supported\n");
			bops->supported_tiles &= ~TILE_X;
		}

		igt_debug("X fence support: %s\n", bool_str(supported));
		if (!supported) {
			bops->supported_hw_tiles &= ~TILE_X;
			bops->linear_to_x = copy_linear_to_x;
			bops->x_to_linear = copy_x_to_linear;
		}
	}

	if (is_hw_tiling_supported(bops, I915_TILING_Y)) {
		bool swizzling_supported;
		bool supported = probe_hw_tiling(bops, I915_TILING_Y,
						 &swizzling_supported);

		if (!swizzling_supported && bops->intel_gen < 12) {
			igt_debug("Swizzling for Y is not supported\n");
			bops->supported_tiles &= ~TILE_Y;
		}

		igt_debug("Y fence support: %s\n", bool_str(supported));
		if (!supported) {
			bops->supported_hw_tiles &= ~TILE_Y;
			bops->linear_to_y = copy_linear_to_y;
			bops->y_to_linear = copy_y_to_linear;
		}
	}

	/* Disable other tiling format functions if not supported */
	if (!is_tiling_supported(bops, I915_TILING_Yf))
		igt_debug("Yf format not supported\n");

	if (!is_tiling_supported(bops, I915_TILING_Ys))
		igt_debug("Ys format not supported\n");

	if (!is_tiling_supported(bops, I915_TILING_4))
		igt_debug("Tile4 format not supported\n");

	if (check_idempotency) {
		idempotency_selftest(bops, I915_TILING_X);
		idempotency_selftest(bops, I915_TILING_Y);
	}

	return bops;
}

/**
 * buf_ops_create
 * @fd: device filedescriptor
 *
 * Create buf_ops structure depending on fd-device capabilities.
 *
 * Returns: opaque pointer to buf_ops.
 *
 */
struct buf_ops *buf_ops_create(int fd)
{
	return __buf_ops_create(fd, false);
}

/**
 * buf_ops_create_with_selftest
 * @fd: device filedescriptor
 *
 * Create buf_ops structure depending on fd-device capabilities.
 * Runs with idempotency selftest to verify software tiling gives same
 * result like hardware tiling (gens with mappable gtt).
 *
 * Returns: opaque pointer to buf_ops.
 *
 */
struct buf_ops *buf_ops_create_with_selftest(int fd)
{
	return __buf_ops_create(fd, true);
}

/**
 * buf_ops_destroy
 * @bops: pointer to buf_ops
 *
 * Function frees buf_ops structure.
 */
void buf_ops_destroy(struct buf_ops *bops)
{
	igt_assert(bops);
	free(bops);
}

/**
 * buf_ops_get_fd
 * @bops: pointer to buf_ops
 *
 * Returns: drm fd
 */
int buf_ops_get_fd(struct buf_ops *bops)
{
	igt_assert(bops);

	return bops->fd;
}

/**
 * buf_ops_get_devid
 * @bops: pointer to buf_ops
 *
 * Returns: device id
 */
uint32_t buf_ops_get_devid(struct buf_ops *bops)
{
	igt_assert(bops);

	return bops->devid;
}

/**
 * buf_ops_get_driver
 * @bops: pointer to buf_ops
 *
 * Returns: intel driver enum value
 */
enum intel_driver buf_ops_get_driver(struct buf_ops *bops)
{
	igt_assert(bops);

	return bops->driver;
}

/**
 * buf_ops_set_software_tiling
 * @bops: pointer to buf_ops
 * @tiling: surface tiling
 * @use_software_tiling: if true use software copying methods, otherwise
 * use hardware (via gtt)
 *
 * Function allows switch X / Y surfaces to software / hardware copying methods
 * which honors tiling and swizzling.
 *
 * Returns:
 * false - switch wasn't possible.
 * true - switch to software / hardware method succeed.
 */
bool buf_ops_set_software_tiling(struct buf_ops *bops,
				 uint32_t tiling,
				 bool use_software_tiling)
{
	bool was_changed = true;

	igt_assert(bops);

	/* Until appropriate code is added we don't support SW tiling on Gen2 */
	if (bops->intel_gen == 2) {
		igt_warn("Change to software tiling on Gen2 is not supported!");
		return false;
	}

	switch (tiling) {
	case I915_TILING_NONE:
		igt_debug("-> use SW on tiling NONE\n");
		break;

	case I915_TILING_X:
		if (use_software_tiling) {
			bool supported = buf_ops_has_tiling_support(bops, tiling);

			igt_assert_f(supported, "Cannot switch to X software tiling\n");
			igt_debug("-> change X to SW\n");
			bops->linear_to_x = copy_linear_to_x;
			bops->x_to_linear = copy_x_to_linear;
		} else {
			if (is_hw_tiling_supported(bops, I915_TILING_X)) {
				igt_debug("-> change X to HW\n");
				bops->linear_to_x = copy_linear_to_gtt;
				bops->x_to_linear = copy_gtt_to_linear;
			} else {
				igt_debug("-> X cannot be changed to HW\n");
				was_changed = false;
			}
		}
		break;

	case I915_TILING_Y:
		if (use_software_tiling) {
			bool supported = buf_ops_has_tiling_support(bops, tiling);

			igt_assert_f(supported, "Cannot switch to Y software tiling\n");
			igt_debug("-> change Y to SW\n");
			bops->linear_to_y = copy_linear_to_y;
			bops->y_to_linear = copy_y_to_linear;
		} else {
			if (is_hw_tiling_supported(bops, I915_TILING_Y)) {
				igt_debug("-> change Y to HW\n");
				bops->linear_to_y = copy_linear_to_gtt;
				bops->y_to_linear = copy_gtt_to_linear;
			} else {
				igt_debug("-> Y cannot be changed to HW\n");
				was_changed = false;
			}
		}
		break;

	case I915_TILING_4:
		igt_debug("-> use SW on tiling 4\n");
		break;

	case I915_TILING_Yf:
		igt_debug("-> use SW on tiling Yf\n");
		break;

	case I915_TILING_Ys:
		igt_debug("-> use SW on tiling Ys\n");
		break;

	default:
		igt_warn("Invalid tiling: %d\n", tiling);
		was_changed = false;
	}

	return was_changed;
}

/**
 * buf_ops_has_hw_fence
 * @bops: pointer to buf_ops
 * @tiling: surface tiling
 *
 * Function checks if surface with tiling has HW fences which can be used
 * to copy it via gtt.
 *
 * Returns:
 * false - fence for tiling is not supported.
 * true - fence for tiling is supported.
 */
bool buf_ops_has_hw_fence(struct buf_ops *bops, uint32_t tiling)
{
	uint32_t tile_mask = TILE_DEF(tiling);

	igt_assert(bops);

	if (tile_mask & bops->supported_hw_tiles)
		return true;

	return false;
}

/**
 * buf_ops_has_tiling_support
 * @bops: pointer to buf_ops
 * @tiling: surface tiling
 *
 * Function checks capabilities to handle surfaces with tiling in GPU.
 *
 * Returns:
 * false - GPU does not support tiling.
 * true - GPU supports tiling.
 */
bool buf_ops_has_tiling_support(struct buf_ops *bops, uint32_t tiling)
{
	uint32_t tile_mask = TILE_DEF(tiling);

	igt_assert(bops);

	if (tile_mask & bops->supported_tiles)
		return true;

	return false;
}