~bkerensa/ubuntu/raring/valgrind/merge-from-deb

/*--------------------------------------------------------------------*/

/*--- MemCheck: Maintain bitmaps of memory, tracking the           ---*/

/*--- accessibility (A) and validity (V) status of each byte.      ---*/

/*---                                                    mc_main.c ---*/

/*--------------------------------------------------------------------*/

/*

   This file is part of MemCheck, a heavyweight Valgrind tool for

   detecting memory errors.

   Copyright (C) 2000-2007 Julian Seward 

      jseward@acm.org

   This program is free software; you can redistribute it and/or

   modify it under the terms of the GNU General Public License as

   published by the Free Software Foundation; either version 2 of the

   License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful, but

   WITHOUT ANY WARRANTY; without even the implied warranty of

   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

   General Public License for more details.

   You should have received a copy of the GNU General Public License

   along with this program; if not, write to the Free Software

   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA

   02111-1307, USA.

   The GNU General Public License is contained in the file COPYING.

*/

#include "pub_tool_basics.h"

#include "pub_tool_aspacemgr.h"

#include "pub_tool_hashtable.h"     // For mc_include.h

#include "pub_tool_libcbase.h"

#include "pub_tool_libcassert.h"

#include "pub_tool_libcprint.h"

#include "pub_tool_machine.h"

#include "pub_tool_mallocfree.h"

#include "pub_tool_options.h"

#include "pub_tool_oset.h"

#include "pub_tool_replacemalloc.h"

#include "pub_tool_tooliface.h"

#include "pub_tool_threadstate.h"

#include "pub_tool_oset.h"

#include "mc_include.h"

#include "memcheck.h"   /* for client requests */

#ifdef HAVE_BUILTIN_EXPECT

#define EXPECTED_TAKEN(cond)     __builtin_expect((cond),1)

#define EXPECTED_NOT_TAKEN(cond) __builtin_expect((cond),0)

#else

#define EXPECTED_TAKEN(cond)     (cond)

#define EXPECTED_NOT_TAKEN(cond) (cond)

#endif

/* Set to 1 to do a little more sanity checking */

#define VG_DEBUG_MEMORY 0

#define DEBUG(fmt, args...) //VG_(printf)(fmt, ## args)

/*------------------------------------------------------------*/

/*--- Fast-case knobs                                      ---*/

/*------------------------------------------------------------*/

// Comment these out to disable the fast cases (don't just set them to zero).

#define PERF_FAST_LOADV    1

#define PERF_FAST_STOREV   1

#define PERF_FAST_SARP     1

#define PERF_FAST_STACK    1

#define PERF_FAST_STACK2   1

/*------------------------------------------------------------*/

/*--- V bits and A bits                                    ---*/

/*------------------------------------------------------------*/

/* Conceptually, every byte value has 8 V bits, which track whether Memcheck

   thinks the corresponding value bit is defined.  And every memory byte

   has an A bit, which tracks whether Memcheck thinks the program can access

   it safely.   So every N-bit register is shadowed with N V bits, and every

   memory byte is shadowed with 8 V bits and one A bit.

   In the implementation, we use two forms of compression (compressed V bits

   and distinguished secondary maps) to avoid the 9-bit-per-byte overhead

   for memory.

   Memcheck also tracks extra information about each heap block that is

   allocated, for detecting memory leaks and other purposes.

*/

/*------------------------------------------------------------*/

/*--- Basic A/V bitmap representation.                     ---*/

/*------------------------------------------------------------*/

/* All reads and writes are checked against a memory map (a.k.a. shadow

   memory), which records the state of all memory in the process.  

   On 32-bit machines the memory map is organised as follows.

   The top 16 bits of an address are used to index into a top-level

   map table, containing 65536 entries.  Each entry is a pointer to a

   second-level map, which records the accesibililty and validity

   permissions for the 65536 bytes indexed by the lower 16 bits of the

   address.  Each byte is represented by two bits (details are below).  So

   each second-level map contains 16384 bytes.  This two-level arrangement

   conveniently divides the 4G address space into 64k lumps, each size 64k

   bytes.

   All entries in the primary (top-level) map must point to a valid

   secondary (second-level) map.  Since many of the 64kB chunks will

   have the same status for every bit -- ie. noaccess (for unused

   address space) or entirely addressable and defined (for code segments) --

   there are three distinguished secondary maps, which indicate 'noaccess',

   'undefined' and 'defined'.  For these uniform 64kB chunks, the primary

   map entry points to the relevant distinguished map.  In practice,

   typically more than half of the addressable memory is represented with

   the 'undefined' or 'defined' distinguished secondary map, so it gives a

   good saving.  It also lets us set the V+A bits of large address regions

   quickly in set_address_range_perms().

   On 64-bit machines it's more complicated.  If we followed the same basic

   scheme we'd have a four-level table which would require too many memory

   accesses.  So instead the top-level map table has 2^19 entries (indexed

   using bits 16..34 of the address);  this covers the bottom 32GB.  Any

   accesses above 32GB are handled with a slow, sparse auxiliary table.

   Valgrind's address space manager tries very hard to keep things below

   this 32GB barrier so that performance doesn't suffer too much.

   Note that this file has a lot of different functions for reading and

   writing shadow memory.  Only a couple are strictly necessary (eg.

   get_vabits2 and set_vabits2), most are just specialised for specific

   common cases to improve performance.

   Aside: the V+A bits are less precise than they could be -- we have no way

   of marking memory as read-only.  It would be great if we could add an

   extra state VA_BITSn_READONLY.  But then we'd have 5 different states,

   which requires 2.3 bits to hold, and there's no way to do that elegantly

   -- we'd have to double up to 4 bits of metadata per byte, which doesn't

   seem worth it.

*/

/* --------------- Basic configuration --------------- */

/* Only change this.  N_PRIMARY_MAP *must* be a power of 2. */

#if VG_WORDSIZE == 4

/* cover the entire address space */

#  define N_PRIMARY_BITS  16

#else

/* Just handle the first 32G fast and the rest via auxiliary

   primaries. */

#  define N_PRIMARY_BITS  19

#endif

/* Do not change this. */

#define N_PRIMARY_MAP  ( ((UWord)1) << N_PRIMARY_BITS)

/* Do not change this. */

#define MAX_PRIMARY_ADDRESS (Addr)((((Addr)65536) * N_PRIMARY_MAP)-1)

/* --------------- Secondary maps --------------- */

// Each byte of memory conceptually has an A bit, which indicates its

// addressability, and 8 V bits, which indicates its definedness.

//

// But because very few bytes are partially defined, we can use a nice

// compression scheme to reduce the size of shadow memory.  Each byte of

// memory has 2 bits which indicates its state (ie. V+A bits):

//

//   00:  noaccess    (unaddressable but treated as fully defined)

//   01:  undefined   (addressable and fully undefined)

//   10:  defined     (addressable and fully defined)

//   11:  partdefined (addressable and partially defined)

//

// In the "partdefined" case, we use a secondary table to store the V bits.

// Each entry in the secondary-V-bits table maps a byte address to its 8 V

// bits.

//

// We store the compressed V+A bits in 8-bit chunks, ie. the V+A bits for

// four bytes (32 bits) of memory are in each chunk.  Hence the name

// "vabits8".  This lets us get the V+A bits for four bytes at a time

// easily (without having to do any shifting and/or masking), and that is a

// very common operation.  (Note that although each vabits8 chunk

// is 8 bits in size, it represents 32 bits of memory.)

//

// The representation is "inverse" little-endian... each 4 bytes of

// memory is represented by a 1 byte value, where:

//

// - the status of byte (a+0) is held in bits [1..0]

// - the status of byte (a+1) is held in bits [3..2]

// - the status of byte (a+2) is held in bits [5..4]

// - the status of byte (a+3) is held in bits [7..6]

//

// It's "inverse" because endianness normally describes a mapping from

// value bits to memory addresses;  in this case the mapping is inverted.

// Ie. instead of particular value bits being held in certain addresses, in

// this case certain addresses are represented by particular value bits.

// See insert_vabits2_into_vabits8() for an example.

// 

// But note that we don't compress the V bits stored in registers;  they

// need to be explicit to made the shadow operations possible.  Therefore

// when moving values between registers and memory we need to convert

// between the expanded in-register format and the compressed in-memory

// format.  This isn't so difficult, it just requires careful attention in a

// few places.

// These represent eight bits of memory.

#define VA_BITS2_NOACCESS     0x0      // 00b

#define VA_BITS2_UNDEFINED    0x1      // 01b

#define VA_BITS2_DEFINED      0x2      // 10b

#define VA_BITS2_PARTDEFINED  0x3      // 11b

// These represent 16 bits of memory.

#define VA_BITS4_NOACCESS     0x0      // 00_00b

#define VA_BITS4_UNDEFINED    0x5      // 01_01b

#define VA_BITS4_DEFINED      0xa      // 10_10b

// These represent 32 bits of memory.

#define VA_BITS8_NOACCESS     0x00     // 00_00_00_00b

#define VA_BITS8_UNDEFINED    0x55     // 01_01_01_01b

#define VA_BITS8_DEFINED      0xaa     // 10_10_10_10b

// These represent 64 bits of memory.

#define VA_BITS16_NOACCESS    0x0000   // 00_00_00_00b x 2

#define VA_BITS16_UNDEFINED   0x5555   // 01_01_01_01b x 2

#define VA_BITS16_DEFINED     0xaaaa   // 10_10_10_10b x 2

#define SM_CHUNKS             16384

#define SM_OFF(aaa)           (((aaa) & 0xffff) >> 2)

#define SM_OFF_16(aaa)        (((aaa) & 0xffff) >> 3)

// Paranoia:  it's critical for performance that the requested inlining

// occurs.  So try extra hard.

#define INLINE    inline __attribute__((always_inline))

static INLINE Addr start_of_this_sm ( Addr a ) {

   return (a & (~SM_MASK));

}

static INLINE Bool is_start_of_sm ( Addr a ) {

   return (start_of_this_sm(a) == a);

}

typedef 

   struct {

      UChar vabits8[SM_CHUNKS];

   }

   SecMap;

// 3 distinguished secondary maps, one for no-access, one for

// accessible but undefined, and one for accessible and defined.

// Distinguished secondaries may never be modified.

#define SM_DIST_NOACCESS   0

#define SM_DIST_UNDEFINED  1

#define SM_DIST_DEFINED    2

static SecMap sm_distinguished[3];

static INLINE Bool is_distinguished_sm ( SecMap* sm ) {

   return sm >= &sm_distinguished[0] && sm <= &sm_distinguished[2];

}

// Forward declaration

static void update_SM_counts(SecMap* oldSM, SecMap* newSM);

/* dist_sm points to one of our three distinguished secondaries.  Make

   a copy of it so that we can write to it.

*/

static SecMap* copy_for_writing ( SecMap* dist_sm )

{

   SecMap* new_sm;

   tl_assert(dist_sm == &sm_distinguished[0]

          || dist_sm == &sm_distinguished[1]

          || dist_sm == &sm_distinguished[2]);

   new_sm = VG_(am_shadow_alloc)(sizeof(SecMap));

   if (new_sm == NULL)

      VG_(out_of_memory_NORETURN)( "memcheck:allocate new SecMap", 

                                   sizeof(SecMap) );

   VG_(memcpy)(new_sm, dist_sm, sizeof(SecMap));

   update_SM_counts(dist_sm, new_sm);

   return new_sm;

}

/* --------------- Stats --------------- */

static Int   n_issued_SMs      = 0;

static Int   n_deissued_SMs    = 0;

static Int   n_noaccess_SMs    = N_PRIMARY_MAP; // start with many noaccess DSMs

static Int   n_undefined_SMs   = 0;

static Int   n_defined_SMs     = 0;

static Int   n_non_DSM_SMs     = 0;

static Int   max_noaccess_SMs  = 0;

static Int   max_undefined_SMs = 0;

static Int   max_defined_SMs   = 0;

static Int   max_non_DSM_SMs   = 0;

/* # searches initiated in auxmap_L1, and # base cmps required */

static ULong n_auxmap_L1_searches  = 0;

static ULong n_auxmap_L1_cmps      = 0;

/* # of searches that missed in auxmap_L1 and therefore had to

   be handed to auxmap_L2. And the number of nodes inserted. */

static ULong n_auxmap_L2_searches  = 0;

static ULong n_auxmap_L2_nodes     = 0;

static Int   n_sanity_cheap     = 0;

static Int   n_sanity_expensive = 0;

static Int   n_secVBit_nodes   = 0;

static Int   max_secVBit_nodes = 0;

static void update_SM_counts(SecMap* oldSM, SecMap* newSM)

{

   if      (oldSM == &sm_distinguished[SM_DIST_NOACCESS ]) n_noaccess_SMs --;

   else if (oldSM == &sm_distinguished[SM_DIST_UNDEFINED]) n_undefined_SMs--;

   else if (oldSM == &sm_distinguished[SM_DIST_DEFINED  ]) n_defined_SMs  --;

   else                                                  { n_non_DSM_SMs  --;

                                                           n_deissued_SMs ++; }

   if      (newSM == &sm_distinguished[SM_DIST_NOACCESS ]) n_noaccess_SMs ++;

   else if (newSM == &sm_distinguished[SM_DIST_UNDEFINED]) n_undefined_SMs++;

   else if (newSM == &sm_distinguished[SM_DIST_DEFINED  ]) n_defined_SMs  ++;

   else                                                  { n_non_DSM_SMs  ++;

                                                           n_issued_SMs   ++; }

   if (n_noaccess_SMs  > max_noaccess_SMs ) max_noaccess_SMs  = n_noaccess_SMs;

   if (n_undefined_SMs > max_undefined_SMs) max_undefined_SMs = n_undefined_SMs;

   if (n_defined_SMs   > max_defined_SMs  ) max_defined_SMs   = n_defined_SMs;

   if (n_non_DSM_SMs   > max_non_DSM_SMs  ) max_non_DSM_SMs   = n_non_DSM_SMs;   

}

/* --------------- Primary maps --------------- */

/* The main primary map.  This covers some initial part of the address

   space, addresses 0 .. (N_PRIMARY_MAP << 16)-1.  The rest of it is

   handled using the auxiliary primary map.  

*/

static SecMap* primary_map[N_PRIMARY_MAP];

/* An entry in the auxiliary primary map.  base must be a 64k-aligned

   value, and sm points at the relevant secondary map.  As with the

   main primary map, the secondary may be either a real secondary, or

   one of the three distinguished secondaries.  DO NOT CHANGE THIS

   LAYOUT: the first word has to be the key for OSet fast lookups.

*/

typedef

   struct { 

      Addr    base;

      SecMap* sm;

   }

   AuxMapEnt;

/* Tunable parameter: How big is the L1 queue? */

#define N_AUXMAP_L1 24

/* Tunable parameter: How far along the L1 queue to insert

   entries resulting from L2 lookups? */

#define AUXMAP_L1_INSERT_IX 12

static struct {

          Addr       base;

          AuxMapEnt* ent; // pointer to the matching auxmap_L2 node

       } 

       auxmap_L1[N_AUXMAP_L1];

static OSet* auxmap_L2 = NULL;

static void init_auxmap_L1_L2 ( void )

{

   Int i;

   for (i = 0; i < N_AUXMAP_L1; i++) {

      auxmap_L1[i].base = 0;

      auxmap_L1[i].ent  = NULL;

   }

   tl_assert(0 == offsetof(AuxMapEnt,base));

   tl_assert(sizeof(Addr) == sizeof(void*));

   auxmap_L2 = VG_(OSetGen_Create)( /*keyOff*/  offsetof(AuxMapEnt,base),

                                    /*fastCmp*/ NULL,

                                    VG_(malloc), VG_(free) );

}

/* Check representation invariants; if OK return NULL; else a

   descriptive bit of text.  Also return the number of

   non-distinguished secondary maps referred to from the auxiliary

   primary maps. */

static HChar* check_auxmap_L1_L2_sanity ( Word* n_secmaps_found )

{

   Word i, j;

   /* On a 32-bit platform, the L2 and L1 tables should

      both remain empty forever.

      On a 64-bit platform:

      In the L2 table:

       all .base & 0xFFFF == 0

       all .base > MAX_PRIMARY_ADDRESS

      In the L1 table:

       all .base & 0xFFFF == 0

       all (.base > MAX_PRIMARY_ADDRESS

            .base & 0xFFFF == 0

            and .ent points to an AuxMapEnt with the same .base)

           or

           (.base == 0 and .ent == NULL)

   */

   *n_secmaps_found = 0;

   if (sizeof(void*) == 4) {

      /* 32-bit platform */

      if (VG_(OSetGen_Size)(auxmap_L2) != 0)

         return "32-bit: auxmap_L2 is non-empty";

      for (i = 0; i < N_AUXMAP_L1; i++) 

        if (auxmap_L1[i].base != 0 || auxmap_L1[i].ent != NULL)

      return "32-bit: auxmap_L1 is non-empty";

   } else {

      /* 64-bit platform */

      UWord elems_seen = 0;

      AuxMapEnt *elem, *res;

      AuxMapEnt key;

      /* L2 table */

      VG_(OSetGen_ResetIter)(auxmap_L2);

      while ( (elem = VG_(OSetGen_Next)(auxmap_L2)) ) {

         elems_seen++;

         if (0 != (elem->base & (Addr)0xFFFF))

            return "64-bit: nonzero .base & 0xFFFF in auxmap_L2";

         if (elem->base <= MAX_PRIMARY_ADDRESS)

            return "64-bit: .base <= MAX_PRIMARY_ADDRESS in auxmap_L2";

         if (elem->sm == NULL)

            return "64-bit: .sm in _L2 is NULL";

         if (!is_distinguished_sm(elem->sm))

            (*n_secmaps_found)++;

      }

      if (elems_seen != n_auxmap_L2_nodes)

         return "64-bit: disagreement on number of elems in _L2";

      /* Check L1-L2 correspondence */

      for (i = 0; i < N_AUXMAP_L1; i++) {

         if (auxmap_L1[i].base == 0 && auxmap_L1[i].ent == NULL)

            continue;

         if (0 != (auxmap_L1[i].base & (Addr)0xFFFF))

            return "64-bit: nonzero .base & 0xFFFF in auxmap_L1";

         if (auxmap_L1[i].base <= MAX_PRIMARY_ADDRESS)

            return "64-bit: .base <= MAX_PRIMARY_ADDRESS in auxmap_L1";

         if (auxmap_L1[i].ent == NULL)

            return "64-bit: .ent is NULL in auxmap_L1";

         if (auxmap_L1[i].ent->base != auxmap_L1[i].base)

            return "64-bit: _L1 and _L2 bases are inconsistent";

         /* Look it up in auxmap_L2. */

         key.base = auxmap_L1[i].base;

         key.sm   = 0;

         res = VG_(OSetGen_Lookup)(auxmap_L2, &key);

         if (res == NULL)

            return "64-bit: _L1 .base not found in _L2";

         if (res != auxmap_L1[i].ent)

            return "64-bit: _L1 .ent disagrees with _L2 entry";

      }

      /* Check L1 contains no duplicates */

      for (i = 0; i < N_AUXMAP_L1; i++) {

         if (auxmap_L1[i].base == 0)

            continue;

	 for (j = i+1; j < N_AUXMAP_L1; j++) {

            if (auxmap_L1[j].base == 0)

               continue;

            if (auxmap_L1[j].base == auxmap_L1[i].base)

               return "64-bit: duplicate _L1 .base entries";

         }

      }

   }

   return NULL; /* ok */

}

static void insert_into_auxmap_L1_at ( Word rank, AuxMapEnt* ent )

{

   Word i;

   tl_assert(ent);

   tl_assert(rank >= 0 && rank < N_AUXMAP_L1);

   for (i = N_AUXMAP_L1-1; i > rank; i--)

      auxmap_L1[i] = auxmap_L1[i-1];

   auxmap_L1[rank].base = ent->base;

   auxmap_L1[rank].ent  = ent;

}

static INLINE AuxMapEnt* maybe_find_in_auxmap ( Addr a )

{

   AuxMapEnt  key;

   AuxMapEnt* res;

   Word       i;

   tl_assert(a > MAX_PRIMARY_ADDRESS);

   a &= ~(Addr)0xFFFF;

   /* First search the front-cache, which is a self-organising

      list containing the most popular entries. */

   if (EXPECTED_TAKEN(auxmap_L1[0].base == a))

      return auxmap_L1[0].ent;

   if (EXPECTED_TAKEN(auxmap_L1[1].base == a)) {

      Addr       t_base = auxmap_L1[0].base;

      AuxMapEnt* t_ent  = auxmap_L1[0].ent;

      auxmap_L1[0].base = auxmap_L1[1].base;

      auxmap_L1[0].ent  = auxmap_L1[1].ent;

      auxmap_L1[1].base = t_base;

      auxmap_L1[1].ent  = t_ent;

      return auxmap_L1[0].ent;

   }

   n_auxmap_L1_searches++;

   for (i = 0; i < N_AUXMAP_L1; i++) {

      if (auxmap_L1[i].base == a) {

         break;

      }

   }

   tl_assert(i >= 0 && i <= N_AUXMAP_L1);

   n_auxmap_L1_cmps += (ULong)(i+1);

   if (i < N_AUXMAP_L1) {

      if (i > 0) {

         Addr       t_base = auxmap_L1[i-1].base;

         AuxMapEnt* t_ent  = auxmap_L1[i-1].ent;

         auxmap_L1[i-1].base = auxmap_L1[i-0].base;

         auxmap_L1[i-1].ent  = auxmap_L1[i-0].ent;

         auxmap_L1[i-0].base = t_base;

         auxmap_L1[i-0].ent  = t_ent;

         i--;

      }

      return auxmap_L1[i].ent;

   }

   n_auxmap_L2_searches++;

   /* First see if we already have it. */

   key.base = a;

   key.sm   = 0;

   res = VG_(OSetGen_Lookup)(auxmap_L2, &key);

   if (res)

      insert_into_auxmap_L1_at( AUXMAP_L1_INSERT_IX, res );

   return res;

}

static AuxMapEnt* find_or_alloc_in_auxmap ( Addr a )

{

   AuxMapEnt *nyu, *res;

   /* First see if we already have it. */

   res = maybe_find_in_auxmap( a );

   if (EXPECTED_TAKEN(res))

      return res;

   /* Ok, there's no entry in the secondary map, so we'll have

      to allocate one. */

   a &= ~(Addr)0xFFFF;

   nyu = (AuxMapEnt*) VG_(OSetGen_AllocNode)( auxmap_L2, sizeof(AuxMapEnt) );

   tl_assert(nyu);

   nyu->base = a;

   nyu->sm   = &sm_distinguished[SM_DIST_NOACCESS];

   VG_(OSetGen_Insert)( auxmap_L2, nyu );

   insert_into_auxmap_L1_at( AUXMAP_L1_INSERT_IX, nyu );

   n_auxmap_L2_nodes++;

   return nyu;

}

/* --------------- SecMap fundamentals --------------- */

// In all these, 'low' means it's definitely in the main primary map,

// 'high' means it's definitely in the auxiliary table.

static INLINE SecMap** get_secmap_low_ptr ( Addr a )

{

   UWord pm_off = a >> 16;

#  if VG_DEBUG_MEMORY >= 1

   tl_assert(pm_off < N_PRIMARY_MAP);

#  endif

   return &primary_map[ pm_off ];

}

static INLINE SecMap** get_secmap_high_ptr ( Addr a )

{

   AuxMapEnt* am = find_or_alloc_in_auxmap(a);

   return &am->sm;

}

static SecMap** get_secmap_ptr ( Addr a )

{

   return ( a <= MAX_PRIMARY_ADDRESS 

          ? get_secmap_low_ptr(a) 

          : get_secmap_high_ptr(a));

}

static INLINE SecMap* get_secmap_for_reading_low ( Addr a )

{

   return *get_secmap_low_ptr(a);

}

static INLINE SecMap* get_secmap_for_reading_high ( Addr a )

{

   return *get_secmap_high_ptr(a);

}

static INLINE SecMap* get_secmap_for_writing_low(Addr a)

{

   SecMap** p = get_secmap_low_ptr(a);

   if (EXPECTED_NOT_TAKEN(is_distinguished_sm(*p)))

      *p = copy_for_writing(*p);

   return *p;

}

static INLINE SecMap* get_secmap_for_writing_high ( Addr a )

{

   SecMap** p = get_secmap_high_ptr(a);

   if (EXPECTED_NOT_TAKEN(is_distinguished_sm(*p)))

      *p = copy_for_writing(*p);

   return *p;

}

/* Produce the secmap for 'a', either from the primary map or by

   ensuring there is an entry for it in the aux primary map.  The

   secmap may be a distinguished one as the caller will only want to

   be able to read it. 

*/

static INLINE SecMap* get_secmap_for_reading ( Addr a )

{

   return ( a <= MAX_PRIMARY_ADDRESS

          ? get_secmap_for_reading_low (a)

          : get_secmap_for_reading_high(a) );

}

/* Produce the secmap for 'a', either from the primary map or by

   ensuring there is an entry for it in the aux primary map.  The

   secmap may not be a distinguished one, since the caller will want

   to be able to write it.  If it is a distinguished secondary, make a

   writable copy of it, install it, and return the copy instead.  (COW

   semantics).

*/

static SecMap* get_secmap_for_writing ( Addr a )

{

   return ( a <= MAX_PRIMARY_ADDRESS

          ? get_secmap_for_writing_low (a)

          : get_secmap_for_writing_high(a) );

}

/* If 'a' has a SecMap, produce it.  Else produce NULL.  But don't

   allocate one if one doesn't already exist.  This is used by the

   leak checker.

*/

static SecMap* maybe_get_secmap_for ( Addr a )

{

   if (a <= MAX_PRIMARY_ADDRESS) {

      return get_secmap_for_reading_low(a);

   } else {

      AuxMapEnt* am = maybe_find_in_auxmap(a);

      return am ? am->sm : NULL;

   }

}

/* --------------- Fundamental functions --------------- */

static INLINE

void insert_vabits2_into_vabits8 ( Addr a, UChar vabits2, UChar* vabits8 )

{

   UInt shift =  (a & 3)  << 1;        // shift by 0, 2, 4, or 6

   *vabits8  &= ~(0x3     << shift);   // mask out the two old bits

   *vabits8  |=  (vabits2 << shift);   // mask  in the two new bits

}

static INLINE

void insert_vabits4_into_vabits8 ( Addr a, UChar vabits4, UChar* vabits8 )

{

   UInt shift;

   tl_assert(VG_IS_2_ALIGNED(a));      // Must be 2-aligned

   shift     =  (a & 2)   << 1;        // shift by 0 or 4

   *vabits8 &= ~(0xf      << shift);   // mask out the four old bits

   *vabits8 |=  (vabits4 << shift);    // mask  in the four new bits

}

static INLINE

UChar extract_vabits2_from_vabits8 ( Addr a, UChar vabits8 )

{

   UInt shift = (a & 3) << 1;          // shift by 0, 2, 4, or 6

   vabits8 >>= shift;                  // shift the two bits to the bottom

   return 0x3 & vabits8;               // mask out the rest

}

static INLINE

UChar extract_vabits4_from_vabits8 ( Addr a, UChar vabits8 )

{

   UInt shift;

   tl_assert(VG_IS_2_ALIGNED(a));      // Must be 2-aligned

   shift = (a & 2) << 1;               // shift by 0 or 4

   vabits8 >>= shift;                  // shift the four bits to the bottom

   return 0xf & vabits8;               // mask out the rest

}

// Note that these four are only used in slow cases.  The fast cases do

// clever things like combine the auxmap check (in

// get_secmap_{read,writ}able) with alignment checks.

// *** WARNING! ***

// Any time this function is called, if it is possible that vabits2

// is equal to VA_BITS2_PARTDEFINED, then the corresponding entry in the

// sec-V-bits table must also be set!

static INLINE

void set_vabits2 ( Addr a, UChar vabits2 )

{

   SecMap* sm       = get_secmap_for_writing(a);

   UWord   sm_off   = SM_OFF(a);

   insert_vabits2_into_vabits8( a, vabits2, &(sm->vabits8[sm_off]) );

}

static INLINE

UChar get_vabits2 ( Addr a )

{

   SecMap* sm       = get_secmap_for_reading(a);

   UWord   sm_off   = SM_OFF(a);

   UChar   vabits8  = sm->vabits8[sm_off];

   return extract_vabits2_from_vabits8(a, vabits8);

}

// *** WARNING! ***

// Any time this function is called, if it is possible that any of the

// 4 2-bit fields in vabits8 are equal to VA_BITS2_PARTDEFINED, then the 

// corresponding entry(s) in the sec-V-bits table must also be set!

static INLINE

UChar get_vabits8_for_aligned_word32 ( Addr a )

{

   SecMap* sm       = get_secmap_for_reading(a);

   UWord   sm_off   = SM_OFF(a);

   UChar   vabits8  = sm->vabits8[sm_off];

   return vabits8;

}

static INLINE

void set_vabits8_for_aligned_word32 ( Addr a, UChar vabits8 )

{

   SecMap* sm       = get_secmap_for_writing(a);

   UWord   sm_off   = SM_OFF(a);

   sm->vabits8[sm_off] = vabits8;

}

// Forward declarations

static UWord get_sec_vbits8(Addr a);

static void  set_sec_vbits8(Addr a, UWord vbits8);

// Returns False if there was an addressability error.

static INLINE

Bool set_vbits8 ( Addr a, UChar vbits8 )

{

   Bool  ok      = True;

   UChar vabits2 = get_vabits2(a);

   if ( VA_BITS2_NOACCESS != vabits2 ) {

      // Addressable.  Convert in-register format to in-memory format.

      // Also remove any existing sec V bit entry for the byte if no

      // longer necessary.

      if      ( V_BITS8_DEFINED   == vbits8 ) { vabits2 = VA_BITS2_DEFINED;   }

      else if ( V_BITS8_UNDEFINED == vbits8 ) { vabits2 = VA_BITS2_UNDEFINED; }

      else                                    { vabits2 = VA_BITS2_PARTDEFINED;

                                                set_sec_vbits8(a, vbits8);  }

      set_vabits2(a, vabits2);

   } else {

      // Unaddressable!  Do nothing -- when writing to unaddressable

      // memory it acts as a black hole, and the V bits can never be seen

      // again.  So we don't have to write them at all.

      ok = False;

   }

   return ok;

}

// Returns False if there was an addressability error.  In that case, we put

// all defined bits into vbits8.

static INLINE

Bool get_vbits8 ( Addr a, UChar* vbits8 )

{

   Bool  ok      = True;

   UChar vabits2 = get_vabits2(a);

   // Convert the in-memory format to in-register format.

   if      ( VA_BITS2_DEFINED   == vabits2 ) { *vbits8 = V_BITS8_DEFINED;   }

   else if ( VA_BITS2_UNDEFINED == vabits2 ) { *vbits8 = V_BITS8_UNDEFINED; }

   else if ( VA_BITS2_NOACCESS  == vabits2 ) {

      *vbits8 = V_BITS8_DEFINED;    // Make V bits defined!

      ok = False;

   } else {

      tl_assert( VA_BITS2_PARTDEFINED == vabits2 );

      *vbits8 = get_sec_vbits8(a);

   }

   return ok;

}

/* --------------- Secondary V bit table ------------ */

// This table holds the full V bit pattern for partially-defined bytes

// (PDBs) that are represented by VA_BITS2_PARTDEFINED in the main shadow

// memory.

//

// Note: the nodes in this table can become stale.  Eg. if you write a PDB,

// then overwrite the same address with a fully defined byte, the sec-V-bit

// node will not necessarily be removed.  This is because checking for

// whether removal is necessary would slow down the fast paths.  

//

// To avoid the stale nodes building up too much, we periodically (once the

// table reaches a certain size) garbage collect (GC) the table by

// traversing it and evicting any "sufficiently stale" nodes, ie. nodes that

// are stale and haven't been touched for a certain number of collections.

// If more than a certain proportion of nodes survived, we increase the

// table size so that GCs occur less often.  

//

// (So this a bit different to a traditional GC, where you definitely want

// to remove any dead nodes.  It's more like we have a resizable cache and

// we're trying to find the right balance how many elements to evict and how

// big to make the cache.)

//

// This policy is designed to avoid bad table bloat in the worst case where

// a program creates huge numbers of stale PDBs -- we would get this bloat

// if we had no GC -- while handling well the case where a node becomes

// stale but shortly afterwards is rewritten with a PDB and so becomes

// non-stale again (which happens quite often, eg. in perf/bz2).  If we just

// remove all stale nodes as soon as possible, we just end up re-adding a

// lot of them in later again.  The "sufficiently stale" approach avoids

// this.  (If a program has many live PDBs, performance will just suck,

// there's no way around that.)

static OSet* secVBitTable;

// Stats

static ULong sec_vbits_new_nodes = 0;

static ULong sec_vbits_updates   = 0;

// This must be a power of two;  this is checked in mc_pre_clo_init().

// The size chosen here is a trade-off:  if the nodes are bigger (ie. cover