[ From: Freddie M ]

Here is a howto for dmalloc using AIX 5L and xlc

For xlc compiler:

1. CC='xlc_r' ./configure --enable-threads

2. settings.h

ifndef LOCK_THREADS
#define LOCK_THREADS 1       <<<<<<<  should be 1
#endif

3. malloc.c changes , diff changes also included

NOTE :  all "malloc" entry points need to be changed to
        have 2 "__" pre and post fix.
        Example:


        malloc     __malloc__
        free       __free__


malloc.c changes:

#ifdef AIX
extern int __multi_threaded;
#include <malloc.h>
#endif

#if LOCK_THREADS
/*
 * mutex lock the malloc library
 */
static  void    lock_thread(void)
{
  /* we only lock if the lock-on counter has reached 0 */
#ifdef AIX
  /* In AIX __mutli_threaded variable is 0 until a thread is created.
   * It will never reset to 0 after that.
   */
  if (__multi_threaded) {
#else
  if (thread_lock_c == 0) {
#endif
#if HAVE_PTHREAD_MUTEX_LOCK
    pthread_mutex_lock(&dmalloc_mutex);
#endif
  }
}

/*
 * mutex unlock the malloc library
 */
static  void    unlock_thread(void)
{
#ifdef AIX
  /* In AIX __mutli_threaded variable is 0 until a thread is created.
   * It will never reset to 0 after that.
   */
  if (__multi_threaded) {
#if HAVE_PTHREAD_MUTEX_UNLOCK
    pthread_mutex_unlock(&dmalloc_mutex);
#endif /* ifdef AIX */
  }

 ...


#ifdef AIX
void __malloc_init__()
{
        pthread_mutex_init(&dmalloc_mutex, THREAD_LOCK_INIT_VAL);
}

void __malloc_prefork_lock__()
{
    pthread_mutex_lock(&dmalloc_mutex);
}

void __malloc_postfork_unlock__()
{
    pthread_mutex_unlock(&dmalloc_mutex);
}


/* legacy function not needed */
int __mallopt__ (int x, int y)
{
}

/* legacy function not needed */
struct mallinfo __mallinfo__()
{
}

/* optional - not really needed */
void __malloc_once__()

{
}

#endif

---- here is the diff -----

*** ../1/dmalloc-4.8.2/malloc.c Mon Feb 26 13:31:16 2001
--- malloc.c    Thu Sep  5 11:26:24 2002
***************
*** 61,66 ****
--- 61,71 ----
  #include <signal.h>
  #endif

+ #ifdef AIX
+ extern int __multi_threaded;
+ #include <malloc.h>
+ #endif
+
  #define DMALLOC_DISABLE

  #include "dmalloc.h"
***************
*** 163,169 ****
--- 168,181 ----
  static        void    lock_thread(void)
  {
    /* we only lock if the lock-on counter has reached 0 */
+ #ifdef AIX
+   /* In AIX __mutli_threaded variable is 0 until a thread is 
created.
+    * It will never reset to 0 after that.
+    */
+   if (__multi_threaded) {
+ #else
    if (thread_lock_c == 0) {
+ #endif
  #if HAVE_PTHREAD_MUTEX_LOCK
      pthread_mutex_lock(&dmalloc_mutex);
  #endif
***************
*** 175,180 ****
--- 187,202 ----
   */
  static        void    unlock_thread(void)
  {
+ #ifdef AIX
+   /* In AIX __mutli_threaded variable is 0 until a thread is 
created.
+    * It will never reset to 0 after that.
+    */
+   if (__multi_threaded) {
+ #if HAVE_PTHREAD_MUTEX_UNLOCK
+     pthread_mutex_unlock(&dmalloc_mutex);
+ #endif
+   }
+ #else
    /* if the lock-on counter has not reached 0 then count it down */
    if (thread_lock_c > 0) {
      thread_lock_c--;
***************
*** 205,210 ****
--- 227,233 ----
      pthread_mutex_unlock(&dmalloc_mutex);
  #endif
    }
+ #endif
  }
  #endif

***************
*** 770,776 ****
   * Returns 0L on error.
   */
  #undef malloc
! DMALLOC_PNT   malloc(DMALLOC_SIZE size)
  {
    char        *file;

--- 793,799 ----
   * Returns 0L on error.
   */
  #undef malloc
! DMALLOC_PNT   __malloc__(DMALLOC_SIZE size)
  {
    char        *file;

***************
*** 785,791 ****
   * Returns 0L on error.
   */
  #undef calloc
! DMALLOC_PNT   calloc(DMALLOC_SIZE num_elements, DMALLOC_SIZE size)
  {
    DMALLOC_SIZE        len = num_elements * size;
    char                *file;
--- 808,814 ----
   * Returns 0L on error.
   */
  #undef calloc
! DMALLOC_PNT   __calloc__(DMALLOC_SIZE num_elements, DMALLOC_SIZE 
size)
  {
    DMALLOC_SIZE        len = num_elements * size;
    char                *file;
***************
*** 804,810 ****
   * Returns 0L on error.
   */
  #undef realloc
! DMALLOC_PNT   realloc(DMALLOC_PNT old_pnt, DMALLOC_SIZE new_size)
  {
    char        *file;

--- 827,833 ----
   * Returns 0L on error.
   */
  #undef realloc
! DMALLOC_PNT   __realloc__(DMALLOC_PNT old_pnt, DMALLOC_SIZE 
new_size)
  {
    char        *file;

***************
*** 901,907 ****
   * is defined by your compiler.
   */
  #undef free
! DMALLOC_FREE_RET      free(DMALLOC_PNT pnt)
  {
    char        *file;
    int ret;
--- 924,930 ----
   * is defined by your compiler.
   */
  #undef free
! DMALLOC_FREE_RET      __free__(DMALLOC_PNT pnt)
  {
    char        *file;
    int ret;
***************
*** 932,937 ****
--- 955,994 ----
    return ret;
  #endif
  }
+
+ #ifdef AIX
+ void __malloc_init__()
+ {
+       pthread_mutex_init(&dmalloc_mutex, THREAD_LOCK_INIT_VAL);
+ }
+
+ void __malloc_prefork_lock__()
+ {
+     pthread_mutex_lock(&dmalloc_mutex);
+ }
+
+ void __malloc_postfork_unlock__()
+ {
+     pthread_mutex_unlock(&dmalloc_mutex);
+ }
+
+
+ /* legacy function not needed */
+ int __mallopt__ (int x, int y)
+ {
+ }
+
+ /* legacy function not needed */
+ struct mallinfo __mallinfo__()
+ {
+ }
+
+ /* optional - not really needed */
+ void __malloc_once__()
+ {
+ }
+
+ #endif

  /******************************* utility calls 
*******************************/

create the file  mem.exp with the following entries

__malloc__
__free__
__realloc__
__calloc__
__mallopt__
__mallinfo__
__malloc_init__
__malloc_prefork_lock__
__malloc_postfork_unlock__
__malloc_once__

Building a shared library for 32 and 64 bit use.

Makefile  - 32 bit  .
1. add -DAIX to DEFS

2. change LIB_TH  . This just actually renames the module
   built by the ld command.
        LIB_TH  = mem32.o

3. Add to CCFLAGS
        CCFLAGS = -g -D_LARGE_THREADS

4. change the $(LIB_TH) build stanza

$(LIB_TH) : $(OBJS) $(THREAD_OBJS)
        ld -b32 -m -o $@ $? -bE:mem.exp -bM:SRE -lpthreads -lc

Makefile - 64 bit

1. add -DAIX to DEFS

2. change LIB_TH  . This just actually renames the module
   built by the ld command.
        LIB_TH  = mem64.o

3. Add to CCFLAGS
        CCFLAGS = -g  -q64 -D_LARGE_THREADS

4. change the $(LIB_TH) build stanza

$(LIB_TH) : $(OBJS) $(THREAD_OBJS)
        ld -b64 -m -o $@ $? -bE:mem.exp -bM:SRE -lpthreads -lc

5. If you care about making a library archive
   Add -X64 to the ar command

$(LIBRARY) : $(OBJS) $(NORMAL_OBJS)
        ar -X64 cr $@ $?
        ranlib $@

Save mem32.o and mem64.o somewhere safe so that a make clean does not
clean them out.

The last step is to make an AIX shared lirary.

	ar -X32_64 -r libmymem.a mem32.o mem64.o
On the AIX machine

	export MALLOCTYPE=user:libmymem.a
	export LIBPATH=<path to where libmymem.a lives>

Define your dmalloc env
Test the library out

Other notes:

In return.h , there is a macro called  #define GET_RET_ADDR(file)   
file = DMALLOC_DEFAULT_FILE
for AIX when the calling address is not know, it will return an 
"unknown" in the log for
unfreed memory

For AIX you can do the following:

typedef struct _frame
{
        struct _frame  *fr_next;        /* frame list */
        int             fr_unused1;
        long            fr_lr;          /* link register */
}
_frame_t;

/* _frame_t *get_stkp(void); */

#pragma mc_func get_stkp \
{ \
        "7c230b78" /* mr 3,1 */ \
}
#pragma reg_killed_by get_stkp gr3

#define CALLER (((_frame_t *)get_stkp())->fr_next->fr_lr)
#define CALLER3 (                                                      
       \
                 ( ((_frame_t *)get_stkp())->fr_next == NULL )         
       \
                 ? DMALLOC_DEFAULT_FILE :                              
       \
                   ( ((_frame_t *)get_stkp())->fr_next->fr_next == 
NULL )     \
                   ? DMALLOC_DEFAULT_FILE : ((_frame_t 
*)get_stkp())->fr_next->fr_next->fr_next->fr_lr \
                )

#define CALLER2 (                                                      
       \
                 ( ((_frame_t *)get_stkp())->fr_next == NULL )         
       \
                 ? DMALLOC_DEFAULT_FILE :                              
       \
                   ( ((_frame_t *)get_stkp())->fr_next->fr_next == 
NULL )     \
                   ? DMALLOC_DEFAULT_FILE : ((_frame_t 
*)get_stkp())->fr_next->fr_next->fr_lr \
                )

You will want to define CALLER2 because if you define CALLER1 you will
just get "malloc" as the return address. You really want to get the
caller that called malloc instead of __malloc__

#define GET_RET_ADDR(file)  file = (char *) CALLER2

or

#define GET_RET_ADDR(file)  file = (char *) CALLER1

Other notes (con't)

1. If you are debugging a system command, say telnet,
   this command may do an exec and not copy over the
   LIBPATH where the damlloc library resides.
   This results in the following error:

USER DEFINED MALLOC ERROR:
Unable to load user supplied object: "libmymem.a(mem32.o)", load() 
errno == 2

   The solution is to place the library in /usr/lib

   chmod 555
   chown bin.bin