.TH LIBCPUSET 3 2006-05-25 "Silicon Graphics, Inc" "SGI" \" -*- nroff -*-
.SH NAME
cpuset, cpuset_version, cpuset_pin, cpuset_size, cpuset_where, cpuset_unpin,
cpuset_alloc, cpuset_free, cpuset_cpus_nbits, cpuset_mems_nbits,
cpuset_setcpus, cpuset_setmems, cpuset_set_iopt, cpuset_set_sopt,
cpuset_getcpus, cpuset_getmems, cpuset_cpus_weight, cpuset_mems_weight,
cpuset_get_iopt, cpuset_get_sopt, cpuset_localcpus, cpuset_localmems,
cpuset_cpumemdist, cpuset_cpu2node, cpuset_addr2node, cpuset_create,
cpuset_delete, cpuset_query, cpuset_modify, cpuset_getcpusetpath,
cpuset_cpusetofpid, cpuset_mountpoint, cpuset_collides_exclusive,
cpuset_nuke, cpuset_init_pidlist, cpuset_pidlist_length,
cpuset_get_pidlist, cpuset_freepidlist, cpuset_move, cpuset_move_all,
cpuset_move_cpuset_tasks,
cpuset_migrate, cpuset_migrate_all, cpuset_reattach,
cpuset_open_memory_pressure, cpuset_read_memory_pressure,
cpuset_close_memory_pressure, cpuset_c_rel_to_sys_cpu,
cpuset_c_sys_to_rel_cpu, cpuset_c_rel_to_sys_mem,
cpuset_c_sys_to_rel_mem, cpuset_p_rel_to_sys_cpu,
cpuset_p_sys_to_rel_cpu, cpuset_p_rel_to_sys_mem,
cpuset_p_sys_to_rel_mem, cpuset_get_placement, cpuset_equal_placement,
cpuset_free_placement, cpuset_fts_open, cpuset_fts_read,
cpuset_fts_reverse, cpuset_fts_rewind, cpuset_fts_get_path,
cpuset_fts_get_stat, cpuset_fts_get_cpuset, cpuset_fts_get_errno,
cpuset_fts_get_info, cpuset_fts_close, cpuset_cpubind,
cpuset_latestcpu, cpuset_membind, cpuset_get_cpu_affinity,
cpuset_set_cpu_affinity, cpuset_get_mem_affinity, cpuset_export,
cpuset_import, cpuset_function
 \- Library for managing CPU and Memory placement.
.SH SYNOPSIS
.nf
.B #include <bitmask.h>
.B #include <cpuset.h>
.sp
.B cc ... -lcpuset
.sp

.SS Basic cpuset routines
.nf
.RI "int cpuset_pin(int " relcpu ");"
.RI "int cpuset_size();"
.RI "int cpuset_where();"
.RI "int cpuset_unpin();"

.SS Basic cpuset routines (Fortran interface)
.nf
.B include 'cpusetf.h'
.BI "integer " relcpu
.sp
.RI "irtn = cpuset_pin(" relcpu ")"
.RI "irtn = cpuset_size()"
.RI "irtn = cpuset_unpin()"
.RI "irtn = cpuset_where()"

.SS Cpuset library version.
.nf
.RI "int cpuset_version();"

.SS Allocate and free `struct cpuset *`:
.nf
.RI "struct cpuset *cpuset_alloc();"
.RI "void cpuset_free(struct cpuset *" cp ");"

.SS Lengths of CPUs and Memory Nodes bitmasks - use to alloc them:
.nf
.RI "int cpuset_cpus_nbits();"
.RI "int cpuset_mems_nbits();"

.SS Set various attributes of a cpuset:
.nf
.RI "int cpuset_setcpus(struct cpuset *" cp ", const struct bitmask *" cpus ");"
.RI "void cpuset_setmems(struct cpuset *" cp ", const struct bitmask *" mems ");"
.RI "int cpuset_set_iopt(struct cpuset *" cp ", const char *" optname ", int " val ");"
.RI "int cpuset_set_sopt(struct cpuset *" cp ", const char *" optname ", const char *" val ");"

.SS Query various attributes of a cpuset:
.nf
.RI "int cpuset_getcpus(const struct cpuset *" cp ", struct bitmask *" cpus ");"
.RI "int cpuset_getmems(const struct cpuset *" cp ", struct bitmask *" mems ");"
.RI "int cpuset_cpus_weight(const struct cpuset *" cp ");"
.RI "int cpuset_mems_weight(const struct cpuset *" cp ");"
.RI "int cpuset_get_iopt(const struct cpuset *" cp ", const char *" optname ");"
.RI "const char *cpuset_get_sopt(const struct cpuset *" cp ", const char *" optname ");"

.SS Local CPUs and Memory Nodes:
.nf
.RI "int cpuset_localcpus(const struct bitmask *" mems ", struct bitmask *" cpus ");"
.RI "int cpuset_localmems(const struct bitmask *" cpus ", struct bitmask *" mems ");"
.RI "unsigned int cpuset_cpumemdist(int " cpu ", int " mem ");"
.RI "int cpuset_cpu2node(int " cpu ");"
.RI "int cpuset_addr2node(void *" addr ");"

.SS Create, delete, query, modify, list and examine cpusets:
.nf
.RI "int cpuset_create(const char *" path ", const struct *" cp ");"
.RI "int cpuset_delete(const char *" path ");"
.RI "int cpuset_query(struct cpuset *" cp ", const char *" path ");"
.RI "int cpuset_modify(const char *" path ", const struct *" cp ");"
.RI "char *cpuset_getcpusetpath(pid_t " pid ", char *" buf ", size_t " size ");"
.RI "int cpuset_cpusetofpid(struct cpuset *" cp ", int " pid ");"
.RI "const char *cpuset_mountpoint();"
.RI "int cpuset_collides_exclusive(const char *" cpusetpath ", const struct cpuset *" cp ");"
.RI "int cpuset_nuke(const char *" path ", unsigned int " seconds ");"

.SS List tasks (pids) currently attached to a cpuset:
.nf
.RI "struct cpuset_pidlist *cpuset_init_pidlist(const char *" path ", int " recurse ");"
.RI "int cpuset_pidlist_length(const struct cpuset_pidlist *" pl ");"
.RI "pid_t cpuset_get_pidlist(const struct cpuset_pidlist *" pl ", int " i ");"
.RI "void cpuset_freepidlist(struct cpuset_pidlist *" pl ");"

.SS Attach tasks to cpusets:
.nf
.RI "int cpuset_move(pid_t p, const char *" path ");"
.RI "int cpuset_move_all(struct cpuset_pid_list *" pl ", const char *" path ");"
.RI "int cpuset_move_cpuset_tasks(const char *" fromrelpath ", const char *" torelpath ");"
.RI "int cpuset_migrate(pid_t " pid ", const char *" path ");"
.RI "int cpuset_migrate_all(struct cpuset_pid_list *" pl ", const char *" path ");"
.RI "int cpuset_reattach(const char *" path ");"

.SS Monitor memory pressure caused by tasks in a cpuset:
.nf
.RI "int cpuset_open_memory_pressure(const char *" cpusetpath ");"
.RI "int cpuset_read_memory_pressure(int " han ");"
.RI "void cpuset_close_memory_pressure(int " han ");"

.SS Converting relative and system CPU and Memory Node numbers:
.nf
.RI "int cpuset_c_rel_to_sys_cpu(const struct cpuset *" cp ", int " relcpu ");"
.RI "int cpuset_c_sys_to_rel_cpu(const struct cpuset *" cp ", int " cpu ");"
.RI "int cpuset_c_rel_to_sys_mem(const struct cpuset *" cp ", int " relmem ");"
.RI "int cpuset_c_sys_to_rel_mem(const struct cpuset *" cp ", int " mem ");"
.RI "int cpuset_p_rel_to_sys_cpu(pid_t " pid ", int " relcpu ");"
.RI "int cpuset_p_sys_to_rel_cpu(pid_t " pid ", int " cpu ");"
.RI "int cpuset_p_rel_to_sys_mem(pid_t " pid ", int " relmem ");"
.RI "int cpuset_p_sys_to_rel_mem(pid_t " pid ", int " mem ");"

.SS Placement operations for detecting cpuset migration:
.nf
.RI "struct cpuset_placement *cpuset_get_placement(pid_t " pid ");"
.RI "int cpuset_equal_placement(const struct cpuset_placement *" plc1 ,
.RI "         const struct cpuset_placement *" plc2 ");"
.RI "void cpuset_free_placement(struct cpuset_placement *" plc ");"

.SS Traverse a cpuset hierarchy:
.nf
.RI "struct cpuset_fts_tree *cpuset_fts_open(const char *" cpusetpath ");"
.RI "const struct cpuset_fts_entry *cpuset_fts_read("
.RI "                   struct cpuset_fts_tree *" cs_tree ");"
.RI "void cpuset_fts_reverse(struct cpuset_fts_tree *" cs_tree ");"
.RI "void cpuset_fts_rewind(struct cpuset_fts_tree *" cs_tree ");"
.RI "const char *cpuset_fts_get_path("
.RI "                   const struct cpuset_fts_entry *" cs_entry ");"
.RI "const struct stat *cpuset_fts_get_stat("
.RI "                   const struct cpuset_fts_entry *" cs_entry ");"
.RI "const struct cpuset *cpuset_fts_get_cpuset("
.RI "                   const struct cpuset_fts_entry *" cs_entry ");"
.RI "int cpuset_fts_get_errno("
.RI "                   const struct cpuset_fts_entry *" cs_entry ");"
.RI "int cpuset_fts_get_info("
.RI "                   const struct cpuset_fts_entry *" cs_entry ");"
.RI "void cpuset_fts_close(struct cpuset_fts_tree *" cs_tree ");"

.SS Bind to a CPU or Memory Node within the current cpuset:
.nf
.RI "int cpuset_cpubind(int " cpu ");"
.RI "int cpuset_latestcpu(pid_t " pid ");"
.RI "int cpuset_membind(int " mem ");"

.SS Export settings to and import from Text Representation:
.nf
.RI "int cpuset_export(const struct cpuset *" cp ", char *" buf ", int " buflen ");"
.RI "int cpuset_import(struct cpuset *" cp ", const char *" file ,
.RI "         int *" errlinenum_ptr ", char *" errmsg_bufptr ", int " errmsg_buflen ");"

.SS Calling [optional] cpuset_* API routines:
.nf
.RI "cpuset_function(const char *" function_name ");"
.fi
.SH DESCRIPTION
.P
Cpusets are named subsets of a systems CPUs and Memory Nodes.
On large SMP and NUMA systems, it is commonly necessary to
control which processes and threads can run on which
CPUs and Memory Nodes.  Such placement can be used to improve
overall system performance, and to manage system utilization
in accordance with appropriate policies.
.P
This cpuset library provides the mechanisms needed to
create, destroy and manage cpusets, and to control the
placement of processes and threads within cpusets.
.P
Though this library has many functions, most applications
using it will only need the few functions in the
.IR "Basic Cpuset API" .
These basic functions enable an application to manage
CPU and Memory placement on SMP and NUMA systems
within existing cpusets.

.P
The
.I Advanced Cpuset API
provides support for creating, destroying and managing
cpusets, and for more complex placement within cpusets.
Workload managers, batch processors and other system
services can use the
.I Advanced Cpuset API
in order to manage cpusets across an entire system,
or a dedicated portion of a system.

.P
All functions are part of the same library, and fully
interoperable.  A system imposed permission model ensures
that one application will not be able to make changes to
the cpusets or placement of other applications, outside of
its currently allowed cpuset.

.SS Basic Cpuset API

The 
.I Basic Cpuset API
provides functions usable from
.B C
for processor and memory placement within a cpuset.

.P
The basic functions enable an application to place various threads
of its execution on specific CPUs within its current cpuset,
and perform related functions such as asking how large the
current cpuset is, and on which CPU within the current cpuset
a thread is currently executing.

.P
The basic
.BI "cpuset_pin(int " relcpu ")"
function uses cpuset relative numbering. In a cpuset of
.B N
CPUs,
.I relcpu
ranges from
.B zero
to
.BR N - 1 .
All functions in the
.I Advanced Cpuset API
use system wide CPU and Memory Node numbering,
to provide a consistent numbering regardless of
cpuset configuration.

.P
Memory placement is done automatically by the basic functions,
local to the requested CPU. Threads may only be pinned on
a single CPU, or unpinned and allowed the run of the entire
current cpuset. This avoids the need to allocate and free the
bitmasks required to specify a set of several CPUs. The basic
functions do not support creating or removing cpusets, only
the placement of threads within an existing cpuset. This
avoids the need to explicitly allocate and free cpuset
structures. Operations only apply to the current thread, avoiding
the need to pass the process id of the thread to be affected.

.SS Basic Functions

.P
The
.I Basic Cpuset API
supports the following functions.


.TP
.BI "int cpuset_pin(int " relcpu ");"
Pin the current task to execute only on the CPU
.IR relcpu ,
which is a relative CPU number within the current cpuset
of that task. Also automatically pin the memory allowed to
be used by the current task to the memory on that same node
(as determined by the advanced
.I cpuset_cpu2node()
function).

.TP
.B int cpuset_size();
Return the number of CPUs in the current tasks cpuset. The
relative CPU numbers that are passed to the
.I cpuset_pin()
function and that are returned by the
.I cpuset_where()
function,
must be between
.B 0
and 
.B "N - 1"
inclusive, where
.B N
is the value
returned by
.IR cpuset_size() .

.TP
.B int cpuset_where();
Return the CPU number, relative to the current tasks
cpuset, of the CPU on which the current task most recently
executed. If a task is allowed to execute on more than
one CPU, then there is no guarantee that the task is still
executing on the CPU returned by cpuset_where, by the time
that the user code obtains the return value.

.TP
.B int cpuset_unpin();
Remove the CPU and Memory pinning affects of any previous
cpuset_pin call, allowing the current task to execute on
any CPU in its current cpuset and to allocate memory on
any Memory Node in its current cpuset.

.SH ERRORS
.TP
.B ENOSYS
Invoked
on an operating system kernel that does not support cpusets.
.TP
.B ENODEV
Invoked on a system that supports
cpusets, but when the cpuset file system is not currently
mounted at
.BR /dev/cpuset .
.TP
.B ENOMEM
Insufficient memory is available.
.TP
.B EBUSY
Attempted
.IR cpuset_delete ()
on a cpuset with attached tasks.
.TP
.B EBUSY
Attempted
.IR cpuset_delete ()
on a cpuset with child cpusets.
.TP
.B ENOENT
Attempted
.IR cpuset_create ()
in a parent cpuset that doesn't exist.
.TP
.B EEXIST
Attempted
.IR cpuset_create ()
for a cpuset that already exists.
.TP
.B EEXIST
Attempted to
.IR rename ()
a cpuset to a name that already exists
.TP
.B ENOTDIR
Attempted to
.IR rename ()
a non-existant cpuset.
.TP
.BR E2BIG
Attempted a
.IR write (2)
system  call on a special cpuset file
with a length larger than some kernel determined upper
limit on the length of such writes.
.TP
.B ESRCH
Attempted to
.IR cpuset_move ()
a non-existance task.
.TP
.B EACCES
Attempted to
.IR cpuset_move ()
a task which one lacks permission to move.
.TP
.B EACCES
Attempted to
.IR write (2)
a
.B memory_pressure
file.
.TP
.B ENOSPC
Attempted to
.IR cpuset_move ()
a task to an empty cpuset.
.TP
.B EINVAL
The 
.B relcpu
argument to 
.IR cpuset_pin ()
is out of range (not between
.BR "zero " and " cpuset_size() - 1" ).
.TP
.B EINVAL
Attempted to change a cpuset in a way that would
violate a cpu_exclusive or mem_exclusive attribute
of that cpuset or any of its siblings.
.TP
.B EINVAL
Attempted to write an empty cpus or mems bitmask
to the kernel.  The kernel creates new cpusets
(via
.IR mkdir )
with empty
.BR cpus
and
.BR mems ,
and the
user level cpuset and bitmask code works with
empty masks.  But the kernel will not allow an
empty bitmask (no bits set) to be written to the
special
.BR cpus
and
.BR mems
files of a cpuset.
.TP
.B EIO
Attempted to
.IR write (2)
a string to a cpuset tasks file that
does not begin with an ASCII decimal
integer.
.TP
.B EIO
Attempted to
.IR rename (2)
a cpuset outside of its current directory.
.TP
.B ENOSPC
Attempted to
.IR write (2)
a list to a cpus
file that did not include any online cpus.
.TP
.B ENOSPC
Attempted to
.IR write (2)
a list to a
.B mems
file that did not include any
online memory nodes.
.TP
.B EACCES
Attempted to add a cpu or mem to a cpuset that is
not already in its parent.
.TP
.B EACCES
Attempted to set cpu_exclusive or mem_exclusive
on a cpuset whose parent lacks the same setting.
.TP
.B ENODEV
The cpuset was removed by another task at the
same time as a
.IR write (2)
was attempted on one of the special files
in the cpuset directory.
.TP
.B EBUSY
Attempted to remove a cpu or mem from a cpuset
that is also in a child of that cpuset.
.TP
.B EFAULT
Attempted to read or write a cpuset file using
a buffer that was outside your accessible address space.
.TP
.B ENAMETOOLONG
Attempted to read a
.BI /proc/ pid /cpuset
file for
a cpuset path that was longer than the kernel page
size.
.TP
.B ENAMETOOLONG
Attempted to create a cpuset whose base directory
name was longer than 255 characters.
.TP
.B ENAMETOOLONG
Attempted to create a cpuset whose full pathname
including "/dev/cpuset" was longer than 4095 characters.
.TP
.B ENXIO
Attempted to create a
.B cpu_exclusive
cpuset whose
.B cpus
covered just part of one or more physical
processor packages, such as including just one of the
two Cores on a package.  For Linux kernel version 2.6.16
on i386 and x86_64, this operation is rejected with this
error to avoid a fatal kernel bug.  Otherwise, this is a
normal and supported operation.
.TP
.B EINVAL
Specified a
.B cpus
or
.B mems
list to the kernel which
included a range with the second number smaller than
the first number.
.TP
.B EINVAL
Specified a
.B cpus
or
.B mems
list to the kernel which
included an invalid character in the string.
.TP
.B ERANGE
Specified a
.B cpus
or
.B mems
list to the kernel which
included a number too large for the kernel to set
in its bitmasks.
.TP
.B ETIME
Time limit for
.BR cpuset_nuke ()
operation reached without successful
completion of operation.
.TP
.B ENOTEMPTY
Tasks remain after multiple attempts by
.BR cpuset_move_cpuset_tasks ()
to move them to a different cpuset.
.TP
.B EPERM
Lacked permission to
.B kill
(send a signal to) a task.
.TP
.B EPERM
Lacked permission to
.B read
a cpuset or its files.
.TP
.B EPERM
Attempted to
.B unlink
a per-cpuset file.  Such files can not be
unlinked.  They can only be removed by removing
.RB ( rmdir )
the
directory representing the cpuset that contains these files.

.SS Advanced Cpuset API

For additional documentation on cpusets, and for
details of the all the other, advanced, routines,
see /usr/share/doc/packages/libcpuset/libcpuset.html and
.br
/usr/share/doc/packages/libbitmask/libbitmask.html.
These same documents are available in plain text format as
/usr/share/doc/packages/libcpuset/libcpuset.txt and
/usr/share/doc/packages/libbitmask/libbitmask.txt.
.SH SEE ALSO
\fBlibbitmask\fP(3)
.SH AUTHOR
pj@sgi.com (Paul Jackson)