LCOV - code coverage report
Current view: directory - memory/jemalloc - jemalloc.c (source / functions) Found Hit Coverage
Test: app.info Lines: 1540 948 61.6 %
Date: 2012-04-07 Functions: 139 79 56.8 %

       1                 : /* -*- Mode: C; tab-width: 8; c-basic-offset: 8; indent-tabs-mode: t -*- */
       2                 : /* vim:set softtabstop=8 shiftwidth=8 noet: */
       3                 : /*-
       4                 :  * Copyright (C) 2006-2008 Jason Evans <jasone@FreeBSD.org>.
       5                 :  * All rights reserved.
       6                 :  *
       7                 :  * Redistribution and use in source and binary forms, with or without
       8                 :  * modification, are permitted provided that the following conditions
       9                 :  * are met:
      10                 :  * 1. Redistributions of source code must retain the above copyright
      11                 :  *    notice(s), this list of conditions and the following disclaimer as
      12                 :  *    the first lines of this file unmodified other than the possible
      13                 :  *    addition of one or more copyright notices.
      14                 :  * 2. Redistributions in binary form must reproduce the above copyright
      15                 :  *    notice(s), this list of conditions and the following disclaimer in
      16                 :  *    the documentation and/or other materials provided with the
      17                 :  *    distribution.
      18                 :  *
      19                 :  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
      20                 :  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
      21                 :  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
      22                 :  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
      23                 :  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
      24                 :  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
      25                 :  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
      26                 :  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
      27                 :  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
      28                 :  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
      29                 :  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
      30                 :  *
      31                 :  *******************************************************************************
      32                 :  *
      33                 :  * This allocator implementation is designed to provide scalable performance
      34                 :  * for multi-threaded programs on multi-processor systems.  The following
      35                 :  * features are included for this purpose:
      36                 :  *
      37                 :  *   + Multiple arenas are used if there are multiple CPUs, which reduces lock
      38                 :  *     contention and cache sloshing.
      39                 :  *
      40                 :  *   + Cache line sharing between arenas is avoided for internal data
      41                 :  *     structures.
      42                 :  *
      43                 :  *   + Memory is managed in chunks and runs (chunks can be split into runs),
      44                 :  *     rather than as individual pages.  This provides a constant-time
      45                 :  *     mechanism for associating allocations with particular arenas.
      46                 :  *
      47                 :  * Allocation requests are rounded up to the nearest size class, and no record
      48                 :  * of the original request size is maintained.  Allocations are broken into
      49                 :  * categories according to size class.  Assuming runtime defaults, 4 kB pages
      50                 :  * and a 16 byte quantum on a 32-bit system, the size classes in each category
      51                 :  * are as follows:
      52                 :  *
      53                 :  *   |=====================================|
      54                 :  *   | Category | Subcategory    |    Size |
      55                 :  *   |=====================================|
      56                 :  *   | Small    | Tiny           |       2 |
      57                 :  *   |          |                |       4 |
      58                 :  *   |          |                |       8 |
      59                 :  *   |          |----------------+---------|
      60                 :  *   |          | Quantum-spaced |      16 |
      61                 :  *   |          |                |      32 |
      62                 :  *   |          |                |      48 |
      63                 :  *   |          |                |     ... |
      64                 :  *   |          |                |     480 |
      65                 :  *   |          |                |     496 |
      66                 :  *   |          |                |     512 |
      67                 :  *   |          |----------------+---------|
      68                 :  *   |          | Sub-page       |    1 kB |
      69                 :  *   |          |                |    2 kB |
      70                 :  *   |=====================================|
      71                 :  *   | Large                     |    4 kB |
      72                 :  *   |                           |    8 kB |
      73                 :  *   |                           |   12 kB |
      74                 :  *   |                           |     ... |
      75                 :  *   |                           | 1012 kB |
      76                 :  *   |                           | 1016 kB |
      77                 :  *   |                           | 1020 kB |
      78                 :  *   |=====================================|
      79                 :  *   | Huge                      |    1 MB |
      80                 :  *   |                           |    2 MB |
      81                 :  *   |                           |    3 MB |
      82                 :  *   |                           |     ... |
      83                 :  *   |=====================================|
      84                 :  *
      85                 :  * NOTE: Due to Mozilla bug 691003, we cannot reserve less than one word for an
      86                 :  * allocation on Linux or Mac.  So on 32-bit *nix, the smallest bucket size is
      87                 :  * 4 bytes, and on 64-bit, the smallest bucket size is 8 bytes.
      88                 :  *
      89                 :  * A different mechanism is used for each category:
      90                 :  *
      91                 :  *   Small : Each size class is segregated into its own set of runs.  Each run
      92                 :  *           maintains a bitmap of which regions are free/allocated.
      93                 :  *
      94                 :  *   Large : Each allocation is backed by a dedicated run.  Metadata are stored
      95                 :  *           in the associated arena chunk header maps.
      96                 :  *
      97                 :  *   Huge : Each allocation is backed by a dedicated contiguous set of chunks.
      98                 :  *          Metadata are stored in a separate red-black tree.
      99                 :  *
     100                 :  *******************************************************************************
     101                 :  */
     102                 : 
     103                 : #ifdef MOZ_MEMORY_ANDROID
     104                 : #define NO_TLS
     105                 : #define _pthread_self() pthread_self()
     106                 : #endif
     107                 : 
     108                 : /*
     109                 :  * On Linux, we use madvise(MADV_DONTNEED) to release memory back to the
     110                 :  * operating system.  If we release 1MB of live pages with MADV_DONTNEED, our
     111                 :  * RSS will decrease by 1MB (almost) immediately.
     112                 :  *
     113                 :  * On Mac, we use madvise(MADV_FREE).  Unlike MADV_DONTNEED on Linux, MADV_FREE
     114                 :  * on Mac doesn't cause the OS to release the specified pages immediately; the
     115                 :  * OS keeps them in our process until the machine comes under memory pressure.
     116                 :  *
     117                 :  * It's therefore difficult to measure the process's RSS on Mac, since, in the
     118                 :  * absence of memory pressure, the contribution from the heap to RSS will not
     119                 :  * decrease due to our madvise calls.
     120                 :  *
     121                 :  * We therefore define MALLOC_DOUBLE_PURGE on Mac.  This causes jemalloc to
     122                 :  * track which pages have been MADV_FREE'd.  You can then call
     123                 :  * jemalloc_purge_freed_pages(), which will force the OS to release those
     124                 :  * MADV_FREE'd pages, making the process's RSS reflect its true memory usage.
     125                 :  *
     126                 :  * The jemalloc_purge_freed_pages definition in jemalloc.h needs to be
     127                 :  * adjusted if MALLOC_DOUBLE_PURGE is ever enabled on Linux.
     128                 :  */
     129                 : #ifdef MOZ_MEMORY_DARWIN
     130                 : #define MALLOC_DOUBLE_PURGE
     131                 : #endif
     132                 : 
     133                 : /*
     134                 :  * MALLOC_PRODUCTION disables assertions and statistics gathering.  It also
     135                 :  * defaults the A and J runtime options to off.  These settings are appropriate
     136                 :  * for production systems.
     137                 :  */
     138                 : #ifndef MOZ_MEMORY_DEBUG
     139                 : #  define       MALLOC_PRODUCTION
     140                 : #endif
     141                 : 
     142                 : /*
     143                 :  * Use only one arena by default.  Mozilla does not currently make extensive
     144                 :  * use of concurrent allocation, so the increased fragmentation associated with
     145                 :  * multiple arenas is not warranted.
     146                 :  */
     147                 : #define MOZ_MEMORY_NARENAS_DEFAULT_ONE
     148                 : 
     149                 : /*
     150                 :  * MALLOC_STATS enables statistics calculation, and is required for
     151                 :  * jemalloc_stats().
     152                 :  */
     153                 : #define MALLOC_STATS
     154                 : 
     155                 : #ifndef MALLOC_PRODUCTION
     156                 :    /*
     157                 :     * MALLOC_DEBUG enables assertions and other sanity checks, and disables
     158                 :     * inline functions.
     159                 :     */
     160                 : #  define MALLOC_DEBUG
     161                 : 
     162                 :    /* Memory filling (junk/zero). */
     163                 : #  define MALLOC_FILL
     164                 : 
     165                 :    /* Allocation tracing. */
     166                 : #  ifndef MOZ_MEMORY_WINDOWS
     167                 : #    define MALLOC_UTRACE
     168                 : #  endif
     169                 : 
     170                 :    /* Support optional abort() on OOM. */
     171                 : #  define MALLOC_XMALLOC
     172                 : 
     173                 :    /* Support SYSV semantics. */
     174                 : #  define MALLOC_SYSV
     175                 : #endif
     176                 : 
     177                 : /*
     178                 :  * MALLOC_VALIDATE causes malloc_usable_size() to perform some pointer
     179                 :  * validation.  There are many possible errors that validation does not even
     180                 :  * attempt to detect.
     181                 :  */
     182                 : #define MALLOC_VALIDATE
     183                 : 
     184                 : /* Embed no-op macros that support memory allocation tracking via valgrind. */
     185                 : #ifdef MOZ_VALGRIND
     186                 : #  define MALLOC_VALGRIND
     187                 : #endif
     188                 : #ifdef MALLOC_VALGRIND
     189                 : #  include <valgrind/valgrind.h>
     190                 : #else
     191                 : #  define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
     192                 : #  define VALGRIND_FREELIKE_BLOCK(addr, rzB)
     193                 : #endif
     194                 : 
     195                 : /*
     196                 :  * MALLOC_BALANCE enables monitoring of arena lock contention and dynamically
     197                 :  * re-balances arena load if exponentially averaged contention exceeds a
     198                 :  * certain threshold.
     199                 :  */
     200                 : /* #define      MALLOC_BALANCE */
     201                 : 
     202                 : /*
     203                 :  * MALLOC_PAGEFILE causes all mmap()ed memory to be backed by temporary
     204                 :  * files, so that if a chunk is mapped, it is guaranteed to be swappable.
     205                 :  * This avoids asynchronous OOM failures that are due to VM over-commit.
     206                 :  */
     207                 : /* #define MALLOC_PAGEFILE */
     208                 : 
     209                 : #ifdef MALLOC_PAGEFILE
     210                 : /* Write size when initializing a page file. */
     211                 : #  define MALLOC_PAGEFILE_WRITE_SIZE 512
     212                 : #endif
     213                 : 
     214                 : #if defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
     215                 : #define _GNU_SOURCE /* For mremap(2). */
     216                 : #define issetugid() 0
     217                 : #if 0 /* Enable in order to test decommit code on Linux. */
     218                 : #  define MALLOC_DECOMMIT
     219                 : #endif
     220                 : #endif
     221                 : 
     222                 : #include <sys/types.h>
     223                 : 
     224                 : #include <errno.h>
     225                 : #include <stdlib.h>
     226                 : #include <limits.h>
     227                 : #include <stdarg.h>
     228                 : #include <stdio.h>
     229                 : #include <string.h>
     230                 : 
     231                 : #ifdef MOZ_MEMORY_WINDOWS
     232                 : 
     233                 : /* Some defines from the CRT internal headers that we need here. */
     234                 : #define _CRT_SPINCOUNT 5000
     235                 : #define __crtInitCritSecAndSpinCount InitializeCriticalSectionAndSpinCount
     236                 : #include <io.h>
     237                 : #include <windows.h>
     238                 : 
     239                 : #pragma warning( disable: 4267 4996 4146 )
     240                 : 
     241                 : #define bool BOOL
     242                 : #define false FALSE
     243                 : #define true TRUE
     244                 : #define inline __inline
     245                 : #define SIZE_T_MAX SIZE_MAX
     246                 : #define STDERR_FILENO 2
     247                 : #define PATH_MAX MAX_PATH
     248                 : #define vsnprintf _vsnprintf
     249                 : 
     250                 : #ifndef NO_TLS
     251                 : static unsigned long tlsIndex = 0xffffffff;
     252                 : #endif 
     253                 : 
     254                 : #define __thread
     255                 : #define _pthread_self() __threadid()
     256                 : #define issetugid() 0
     257                 : 
     258                 : /* use MSVC intrinsics */
     259                 : #pragma intrinsic(_BitScanForward)
     260                 : static __forceinline int
     261                 : ffs(int x)
     262                 : {
     263                 :         unsigned long i;
     264                 : 
     265                 :         if (_BitScanForward(&i, x) != 0)
     266                 :                 return (i + 1);
     267                 : 
     268                 :         return (0);
     269                 : }
     270                 : 
     271                 : /* Implement getenv without using malloc */
     272                 : static char mozillaMallocOptionsBuf[64];
     273                 : 
     274                 : #define getenv xgetenv
     275                 : static char *
     276                 : getenv(const char *name)
     277                 : {
     278                 : 
     279                 :         if (GetEnvironmentVariableA(name, (LPSTR)&mozillaMallocOptionsBuf,
     280                 :                     sizeof(mozillaMallocOptionsBuf)) > 0)
     281                 :                 return (mozillaMallocOptionsBuf);
     282                 : 
     283                 :         return (NULL);
     284                 : }
     285                 : 
     286                 : typedef unsigned char uint8_t;
     287                 : typedef unsigned uint32_t;
     288                 : typedef unsigned long long uint64_t;
     289                 : typedef unsigned long long uintmax_t;
     290                 : #if defined(MOZ_MEMORY_SIZEOF_PTR_2POW) && (MOZ_MEMORY_SIZEOF_PTR_2POW == 3)
     291                 : typedef long long ssize_t;
     292                 : #else
     293                 : typedef long ssize_t;
     294                 : #endif
     295                 : 
     296                 : #define MALLOC_DECOMMIT
     297                 : #endif
     298                 : 
     299                 : #ifndef MOZ_MEMORY_WINDOWS
     300                 : #ifndef MOZ_MEMORY_SOLARIS
     301                 : #include <sys/cdefs.h>
     302                 : #endif
     303                 : #ifndef __DECONST
     304                 : #  define __DECONST(type, var)  ((type)(uintptr_t)(const void *)(var))
     305                 : #endif
     306                 : #ifndef MOZ_MEMORY
     307                 : __FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 180599 2008-07-18 19:35:44Z jasone $");
     308                 : #include "libc_private.h"
     309                 : #ifdef MALLOC_DEBUG
     310                 : #  define _LOCK_DEBUG
     311                 : #endif
     312                 : #include "spinlock.h"
     313                 : #include "namespace.h"
     314                 : #endif
     315                 : #include <sys/mman.h>
     316                 : #ifndef MADV_FREE
     317                 : #  define MADV_FREE     MADV_DONTNEED
     318                 : #endif
     319                 : #ifndef MAP_NOSYNC
     320                 : #  define MAP_NOSYNC    0
     321                 : #endif
     322                 : #include <sys/param.h>
     323                 : #ifndef MOZ_MEMORY
     324                 : #include <sys/stddef.h>
     325                 : #endif
     326                 : #include <sys/time.h>
     327                 : #include <sys/types.h>
     328                 : #if !defined(MOZ_MEMORY_SOLARIS) && !defined(MOZ_MEMORY_ANDROID)
     329                 : #include <sys/sysctl.h>
     330                 : #endif
     331                 : #include <sys/uio.h>
     332                 : #ifndef MOZ_MEMORY
     333                 : #include <sys/ktrace.h> /* Must come after several other sys/ includes. */
     334                 : 
     335                 : #include <machine/atomic.h>
     336                 : #include <machine/cpufunc.h>
     337                 : #include <machine/vmparam.h>
     338                 : #endif
     339                 : 
     340                 : #include <errno.h>
     341                 : #include <limits.h>
     342                 : #ifndef SIZE_T_MAX
     343                 : #  define SIZE_T_MAX    SIZE_MAX
     344                 : #endif
     345                 : #include <pthread.h>
     346                 : #ifdef MOZ_MEMORY_DARWIN
     347                 : #define _pthread_self pthread_self
     348                 : #define _pthread_mutex_init pthread_mutex_init
     349                 : #define _pthread_mutex_trylock pthread_mutex_trylock
     350                 : #define _pthread_mutex_lock pthread_mutex_lock
     351                 : #define _pthread_mutex_unlock pthread_mutex_unlock
     352                 : #endif
     353                 : #include <sched.h>
     354                 : #include <stdarg.h>
     355                 : #include <stdio.h>
     356                 : #include <stdbool.h>
     357                 : #include <stdint.h>
     358                 : #include <stdlib.h>
     359                 : #include <string.h>
     360                 : #ifndef MOZ_MEMORY_DARWIN
     361                 : #include <strings.h>
     362                 : #endif
     363                 : #include <unistd.h>
     364                 : 
     365                 : #ifdef MOZ_MEMORY_DARWIN
     366                 : #include <libkern/OSAtomic.h>
     367                 : #include <mach/mach_error.h>
     368                 : #include <mach/mach_init.h>
     369                 : #include <mach/vm_map.h>
     370                 : #include <malloc/malloc.h>
     371                 : #endif
     372                 : 
     373                 : #ifndef MOZ_MEMORY
     374                 : #include "un-namespace.h"
     375                 : #endif
     376                 : 
     377                 : #endif
     378                 : 
     379                 : #include "jemalloc_types.h"
     380                 : #include "linkedlist.h"
     381                 : 
     382                 : /* Some tools, such as /dev/dsp wrappers, LD_PRELOAD libraries that
     383                 :  * happen to override mmap() and call dlsym() from their overridden
     384                 :  * mmap(). The problem is that dlsym() calls malloc(), and this ends
     385                 :  * up in a dead lock in jemalloc.
     386                 :  * On these systems, we prefer to directly use the system call.
     387                 :  * We do that for Linux systems and kfreebsd with GNU userland.
     388                 :  * Note sanity checks are not done (alignment of offset, ...) because
     389                 :  * the uses of mmap are pretty limited, in jemalloc.
     390                 :  *
     391                 :  * On Alpha, glibc has a bug that prevents syscall() to work for system
     392                 :  * calls with 6 arguments
     393                 :  */
     394                 : #if (defined(MOZ_MEMORY_LINUX) && !defined(__alpha__)) || \
     395                 :     (defined(MOZ_MEMORY_BSD) && defined(__GLIBC__))
     396                 : #include <sys/syscall.h>
     397                 : #if defined(SYS_mmap) || defined(SYS_mmap2)
     398                 : static inline
     399           64774 : void *_mmap(void *addr, size_t length, int prot, int flags,
     400                 :             int fd, off_t offset)
     401                 : {
     402                 : /* S390 only passes one argument to the mmap system call, which is a
     403                 :  * pointer to a structure containing the arguments */
     404                 : #ifdef __s390__
     405                 :         struct {
     406                 :                 void *addr;
     407                 :                 size_t length;
     408                 :                 int prot;
     409                 :                 int flags;
     410                 :                 int fd;
     411                 :                 off_t offset;
     412                 :         } args = { addr, length, prot, flags, fd, offset };
     413                 :         return (void *) syscall(SYS_mmap, &args);
     414                 : #else
     415                 : #ifdef SYS_mmap2
     416           64774 :         return (void *) syscall(SYS_mmap2, addr, length, prot, flags,
     417                 :                                fd, offset >> 12);
     418                 : #else
     419                 :         return (void *) syscall(SYS_mmap, addr, length, prot, flags,
     420                 :                                fd, offset);
     421                 : #endif
     422                 : #endif
     423                 : }
     424                 : #define mmap _mmap
     425                 : #define munmap(a, l) syscall(SYS_munmap, a, l)
     426                 : #endif
     427                 : #endif
     428                 : 
     429                 : #ifdef MOZ_MEMORY_DARWIN
     430                 : static const bool isthreaded = true;
     431                 : #endif
     432                 : 
     433                 : #if defined(MOZ_MEMORY_SOLARIS) && defined(MAP_ALIGN) && !defined(JEMALLOC_NEVER_USES_MAP_ALIGN)
     434                 : #define JEMALLOC_USES_MAP_ALIGN  /* Required on Solaris 10. Might improve performance elsewhere. */
     435                 : #endif
     436                 : 
     437                 : #define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
     438                 : 
     439                 : #ifdef MOZ_MEMORY_WINDOWS
     440                 :    /* MSVC++ does not support C99 variable-length arrays. */
     441                 : #  define RB_NO_C99_VARARRAYS
     442                 : #endif
     443                 : #include "rb.h"
     444                 : 
     445                 : #ifdef MALLOC_DEBUG
     446                 :    /* Disable inlining to make debugging easier. */
     447                 : #ifdef inline
     448                 : #undef inline
     449                 : #endif
     450                 : 
     451                 : #  define inline
     452                 : #endif
     453                 : 
     454                 : /* Size of stack-allocated buffer passed to strerror_r(). */
     455                 : #define STRERROR_BUF            64
     456                 : 
     457                 : /* Minimum alignment of non-tiny allocations is 2^QUANTUM_2POW_MIN bytes. */
     458                 : #  define QUANTUM_2POW_MIN      4
     459                 : #ifdef MOZ_MEMORY_SIZEOF_PTR_2POW
     460                 : #  define SIZEOF_PTR_2POW               MOZ_MEMORY_SIZEOF_PTR_2POW
     461                 : #else
     462                 : #  define SIZEOF_PTR_2POW       2
     463                 : #endif
     464                 : #define PIC
     465                 : #ifndef MOZ_MEMORY_DARWIN
     466                 : static const bool isthreaded = true;
     467                 : #else
     468                 : #  define NO_TLS
     469                 : #endif
     470                 : #if 0
     471                 : #ifdef __i386__
     472                 : #  define QUANTUM_2POW_MIN      4
     473                 : #  define SIZEOF_PTR_2POW       2
     474                 : #  define CPU_SPINWAIT          __asm__ volatile("pause")
     475                 : #endif
     476                 : #ifdef __ia64__
     477                 : #  define QUANTUM_2POW_MIN      4
     478                 : #  define SIZEOF_PTR_2POW       3
     479                 : #endif
     480                 : #ifdef __alpha__
     481                 : #  define QUANTUM_2POW_MIN      4
     482                 : #  define SIZEOF_PTR_2POW       3
     483                 : #  define NO_TLS
     484                 : #endif
     485                 : #ifdef __sparc64__
     486                 : #  define QUANTUM_2POW_MIN      4
     487                 : #  define SIZEOF_PTR_2POW       3
     488                 : #  define NO_TLS
     489                 : #endif
     490                 : #ifdef __amd64__
     491                 : #  define QUANTUM_2POW_MIN      4
     492                 : #  define SIZEOF_PTR_2POW       3
     493                 : #  define CPU_SPINWAIT          __asm__ volatile("pause")
     494                 : #endif
     495                 : #ifdef __arm__
     496                 : #  define QUANTUM_2POW_MIN      3
     497                 : #  define SIZEOF_PTR_2POW       2
     498                 : #  define NO_TLS
     499                 : #endif
     500                 : #ifdef __mips__
     501                 : #  define QUANTUM_2POW_MIN      3
     502                 : #  define SIZEOF_PTR_2POW       2
     503                 : #  define NO_TLS
     504                 : #endif
     505                 : #ifdef __powerpc__
     506                 : #  define QUANTUM_2POW_MIN      4
     507                 : #  define SIZEOF_PTR_2POW       2
     508                 : #endif
     509                 : #endif
     510                 : 
     511                 : #define SIZEOF_PTR              (1U << SIZEOF_PTR_2POW)
     512                 : 
     513                 : /* sizeof(int) == (1U << SIZEOF_INT_2POW). */
     514                 : #ifndef SIZEOF_INT_2POW
     515                 : #  define SIZEOF_INT_2POW       2
     516                 : #endif
     517                 : 
     518                 : /* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
     519                 : #if (!defined(PIC) && !defined(NO_TLS))
     520                 : #  define NO_TLS
     521                 : #endif
     522                 : 
     523                 : #ifdef NO_TLS
     524                 :    /* MALLOC_BALANCE requires TLS. */
     525                 : #  ifdef MALLOC_BALANCE
     526                 : #    undef MALLOC_BALANCE
     527                 : #  endif
     528                 : #endif
     529                 : 
     530                 : /*
     531                 :  * Size and alignment of memory chunks that are allocated by the OS's virtual
     532                 :  * memory system.
     533                 :  */
     534                 : #define CHUNK_2POW_DEFAULT      20
     535                 : /* Maximum number of dirty pages per arena. */
     536                 : #define DIRTY_MAX_DEFAULT       (1U << 10)
     537                 : 
     538                 : /*
     539                 :  * Maximum size of L1 cache line.  This is used to avoid cache line aliasing,
     540                 :  * so over-estimates are okay (up to a point), but under-estimates will
     541                 :  * negatively affect performance.
     542                 :  */
     543                 : #define CACHELINE_2POW          6
     544                 : #define CACHELINE               ((size_t)(1U << CACHELINE_2POW))
     545                 : 
     546                 : /*
     547                 :  * Smallest size class to support.  On Linux and Mac, even malloc(1) must
     548                 :  * reserve a word's worth of memory (see Mozilla bug 691003).
     549                 :  */
     550                 : #ifdef MOZ_MEMORY_WINDOWS
     551                 : #define TINY_MIN_2POW           1
     552                 : #else
     553                 : #define TINY_MIN_2POW           (sizeof(void*) == 8 ? 3 : 2)
     554                 : #endif
     555                 : 
     556                 : /*
     557                 :  * Maximum size class that is a multiple of the quantum, but not (necessarily)
     558                 :  * a power of 2.  Above this size, allocations are rounded up to the nearest
     559                 :  * power of 2.
     560                 :  */
     561                 : #define SMALL_MAX_2POW_DEFAULT  9
     562                 : #define SMALL_MAX_DEFAULT       (1U << SMALL_MAX_2POW_DEFAULT)
     563                 : 
     564                 : /*
     565                 :  * RUN_MAX_OVRHD indicates maximum desired run header overhead.  Runs are sized
     566                 :  * as small as possible such that this setting is still honored, without
     567                 :  * violating other constraints.  The goal is to make runs as small as possible
     568                 :  * without exceeding a per run external fragmentation threshold.
     569                 :  *
     570                 :  * We use binary fixed point math for overhead computations, where the binary
     571                 :  * point is implicitly RUN_BFP bits to the left.
     572                 :  *
     573                 :  * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
     574                 :  * honored for some/all object sizes, since there is one bit of header overhead
     575                 :  * per object (plus a constant).  This constraint is relaxed (ignored) for runs
     576                 :  * that are so small that the per-region overhead is greater than:
     577                 :  *
     578                 :  *   (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
     579                 :  */
     580                 : #define RUN_BFP                 12
     581                 : /*                                    \/   Implicit binary fixed point. */
     582                 : #define RUN_MAX_OVRHD           0x0000003dU
     583                 : #define RUN_MAX_OVRHD_RELAX     0x00001800U
     584                 : 
     585                 : /* Put a cap on small object run size.  This overrides RUN_MAX_OVRHD. */
     586                 : #define RUN_MAX_SMALL_2POW      15
     587                 : #define RUN_MAX_SMALL           (1U << RUN_MAX_SMALL_2POW)
     588                 : 
     589                 : /*
     590                 :  * Hyper-threaded CPUs may need a special instruction inside spin loops in
     591                 :  * order to yield to another virtual CPU.  If no such instruction is defined
     592                 :  * above, make CPU_SPINWAIT a no-op.
     593                 :  */
     594                 : #ifndef CPU_SPINWAIT
     595                 : #  define CPU_SPINWAIT
     596                 : #endif
     597                 : 
     598                 : /*
     599                 :  * Adaptive spinning must eventually switch to blocking, in order to avoid the
     600                 :  * potential for priority inversion deadlock.  Backing off past a certain point
     601                 :  * can actually waste time.
     602                 :  */
     603                 : #define SPIN_LIMIT_2POW         11
     604                 : 
     605                 : /*
     606                 :  * Conversion from spinning to blocking is expensive; we use (1U <<
     607                 :  * BLOCK_COST_2POW) to estimate how many more times costly blocking is than
     608                 :  * worst-case spinning.
     609                 :  */
     610                 : #define BLOCK_COST_2POW         4
     611                 : 
     612                 : #ifdef MALLOC_BALANCE
     613                 :    /*
     614                 :     * We use an exponential moving average to track recent lock contention,
     615                 :     * where the size of the history window is N, and alpha=2/(N+1).
     616                 :     *
     617                 :     * Due to integer math rounding, very small values here can cause
     618                 :     * substantial degradation in accuracy, thus making the moving average decay
     619                 :     * faster than it would with precise calculation.
     620                 :     */
     621                 : #  define BALANCE_ALPHA_INV_2POW        9
     622                 : 
     623                 :    /*
     624                 :     * Threshold value for the exponential moving contention average at which to
     625                 :     * re-assign a thread.
     626                 :     */
     627                 : #  define BALANCE_THRESHOLD_DEFAULT     (1U << (SPIN_LIMIT_2POW-4))
     628                 : #endif
     629                 : 
     630                 : /******************************************************************************/
     631                 : 
     632                 : /* MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. */
     633                 : #if defined(MALLOC_DECOMMIT) && defined(MALLOC_DOUBLE_PURGE)
     634                 : #error MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive.
     635                 : #endif
     636                 : 
     637                 : /*
     638                 :  * Mutexes based on spinlocks.  We can't use normal pthread spinlocks in all
     639                 :  * places, because they require malloc()ed memory, which causes bootstrapping
     640                 :  * issues in some cases.
     641                 :  */
     642                 : #if defined(MOZ_MEMORY_WINDOWS)
     643                 : #define malloc_mutex_t CRITICAL_SECTION
     644                 : #define malloc_spinlock_t CRITICAL_SECTION
     645                 : #elif defined(MOZ_MEMORY_DARWIN)
     646                 : typedef struct {
     647                 :         OSSpinLock      lock;
     648                 : } malloc_mutex_t;
     649                 : typedef struct {
     650                 :         OSSpinLock      lock;
     651                 : } malloc_spinlock_t;
     652                 : #elif defined(MOZ_MEMORY)
     653                 : typedef pthread_mutex_t malloc_mutex_t;
     654                 : typedef pthread_mutex_t malloc_spinlock_t;
     655                 : #else
     656                 : /* XXX these should #ifdef these for freebsd (and linux?) only */
     657                 : typedef struct {
     658                 :         spinlock_t      lock;
     659                 : } malloc_mutex_t;
     660                 : typedef malloc_spinlock_t malloc_mutex_t;
     661                 : #endif
     662                 : 
     663                 : /* Set to true once the allocator has been initialized. */
     664                 : static bool malloc_initialized = false;
     665                 : 
     666                 : #if defined(MOZ_MEMORY_WINDOWS)
     667                 : /* No init lock for Windows. */
     668                 : #elif defined(MOZ_MEMORY_DARWIN)
     669                 : static malloc_mutex_t init_lock = {OS_SPINLOCK_INIT};
     670                 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
     671                 : static malloc_mutex_t init_lock = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP;
     672                 : #elif defined(MOZ_MEMORY)
     673                 : static malloc_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
     674                 : #else
     675                 : static malloc_mutex_t init_lock = {_SPINLOCK_INITIALIZER};
     676                 : #endif
     677                 : 
     678                 : /******************************************************************************/
     679                 : /*
     680                 :  * Statistics data structures.
     681                 :  */
     682                 : 
     683                 : #ifdef MALLOC_STATS
     684                 : 
     685                 : typedef struct malloc_bin_stats_s malloc_bin_stats_t;
     686                 : struct malloc_bin_stats_s {
     687                 :         /*
     688                 :          * Number of allocation requests that corresponded to the size of this
     689                 :          * bin.
     690                 :          */
     691                 :         uint64_t        nrequests;
     692                 : 
     693                 :         /* Total number of runs created for this bin's size class. */
     694                 :         uint64_t        nruns;
     695                 : 
     696                 :         /*
     697                 :          * Total number of runs reused by extracting them from the runs tree for
     698                 :          * this bin's size class.
     699                 :          */
     700                 :         uint64_t        reruns;
     701                 : 
     702                 :         /* High-water mark for this bin. */
     703                 :         unsigned long   highruns;
     704                 : 
     705                 :         /* Current number of runs in this bin. */
     706                 :         unsigned long   curruns;
     707                 : };
     708                 : 
     709                 : typedef struct arena_stats_s arena_stats_t;
     710                 : struct arena_stats_s {
     711                 :         /* Number of bytes currently mapped. */
     712                 :         size_t          mapped;
     713                 : 
     714                 :         /*
     715                 :          * Total number of purge sweeps, total number of madvise calls made,
     716                 :          * and total pages purged in order to keep dirty unused memory under
     717                 :          * control.
     718                 :          */
     719                 :         uint64_t        npurge;
     720                 :         uint64_t        nmadvise;
     721                 :         uint64_t        purged;
     722                 : #ifdef MALLOC_DECOMMIT
     723                 :         /*
     724                 :          * Total number of decommit/commit operations, and total number of
     725                 :          * pages decommitted.
     726                 :          */
     727                 :         uint64_t        ndecommit;
     728                 :         uint64_t        ncommit;
     729                 :         uint64_t        decommitted;
     730                 : #endif
     731                 : 
     732                 :         /* Current number of committed pages. */
     733                 :         size_t          committed;
     734                 : 
     735                 :         /* Per-size-category statistics. */
     736                 :         size_t          allocated_small;
     737                 :         uint64_t        nmalloc_small;
     738                 :         uint64_t        ndalloc_small;
     739                 : 
     740                 :         size_t          allocated_large;
     741                 :         uint64_t        nmalloc_large;
     742                 :         uint64_t        ndalloc_large;
     743                 : 
     744                 : #ifdef MALLOC_BALANCE
     745                 :         /* Number of times this arena reassigned a thread due to contention. */
     746                 :         uint64_t        nbalance;
     747                 : #endif
     748                 : };
     749                 : 
     750                 : typedef struct chunk_stats_s chunk_stats_t;
     751                 : struct chunk_stats_s {
     752                 :         /* Number of chunks that were allocated. */
     753                 :         uint64_t        nchunks;
     754                 : 
     755                 :         /* High-water mark for number of chunks allocated. */
     756                 :         unsigned long   highchunks;
     757                 : 
     758                 :         /*
     759                 :          * Current number of chunks allocated.  This value isn't maintained for
     760                 :          * any other purpose, so keep track of it in order to be able to set
     761                 :          * highchunks.
     762                 :          */
     763                 :         unsigned long   curchunks;
     764                 : };
     765                 : 
     766                 : #endif /* #ifdef MALLOC_STATS */
     767                 : 
     768                 : /******************************************************************************/
     769                 : /*
     770                 :  * Extent data structures.
     771                 :  */
     772                 : 
     773                 : /* Tree of extents. */
     774                 : typedef struct extent_node_s extent_node_t;
     775                 : struct extent_node_s {
     776                 :         /* Linkage for the size/address-ordered tree. */
     777                 :         rb_node(extent_node_t) link_szad;
     778                 : 
     779                 :         /* Linkage for the address-ordered tree. */
     780                 :         rb_node(extent_node_t) link_ad;
     781                 : 
     782                 :         /* Pointer to the extent that this tree node is responsible for. */
     783                 :         void    *addr;
     784                 : 
     785                 :         /* Total region size. */
     786                 :         size_t  size;
     787                 : };
     788                 : typedef rb_tree(extent_node_t) extent_tree_t;
     789                 : 
     790                 : /******************************************************************************/
     791                 : /*
     792                 :  * Radix tree data structures.
     793                 :  */
     794                 : 
     795                 : #ifdef MALLOC_VALIDATE
     796                 :    /*
     797                 :     * Size of each radix tree node (must be a power of 2).  This impacts tree
     798                 :     * depth.
     799                 :     */
     800                 : #  if (SIZEOF_PTR == 4)
     801                 : #    define MALLOC_RTREE_NODESIZE (1U << 14)
     802                 : #  else
     803                 : #    define MALLOC_RTREE_NODESIZE CACHELINE
     804                 : #  endif
     805                 : 
     806                 : typedef struct malloc_rtree_s malloc_rtree_t;
     807                 : struct malloc_rtree_s {
     808                 :         malloc_spinlock_t       lock;
     809                 :         void                    **root;
     810                 :         unsigned                height;
     811                 :         unsigned                level2bits[1]; /* Dynamically sized. */
     812                 : };
     813                 : #endif
     814                 : 
     815                 : /******************************************************************************/
     816                 : /*
     817                 :  * Arena data structures.
     818                 :  */
     819                 : 
     820                 : typedef struct arena_s arena_t;
     821                 : typedef struct arena_bin_s arena_bin_t;
     822                 : 
     823                 : /* Each element of the chunk map corresponds to one page within the chunk. */
     824                 : typedef struct arena_chunk_map_s arena_chunk_map_t;
     825                 : struct arena_chunk_map_s {
     826                 :         /*
     827                 :          * Linkage for run trees.  There are two disjoint uses:
     828                 :          *
     829                 :          * 1) arena_t's runs_avail tree.
     830                 :          * 2) arena_run_t conceptually uses this linkage for in-use non-full
     831                 :          *    runs, rather than directly embedding linkage.
     832                 :          */
     833                 :         rb_node(arena_chunk_map_t)      link;
     834                 : 
     835                 :         /*
     836                 :          * Run address (or size) and various flags are stored together.  The bit
     837                 :          * layout looks like (assuming 32-bit system):
     838                 :          *
     839                 :          *   ???????? ???????? ????---- -mckdzla
     840                 :          *
     841                 :          * ? : Unallocated: Run address for first/last pages, unset for internal
     842                 :          *                  pages.
     843                 :          *     Small: Run address.
     844                 :          *     Large: Run size for first page, unset for trailing pages.
     845                 :          * - : Unused.
     846                 :          * m : MADV_FREE/MADV_DONTNEED'ed?
     847                 :          * c : decommitted?
     848                 :          * k : key?
     849                 :          * d : dirty?
     850                 :          * z : zeroed?
     851                 :          * l : large?
     852                 :          * a : allocated?
     853                 :          *
     854                 :          * Following are example bit patterns for the three types of runs.
     855                 :          *
     856                 :          * r : run address
     857                 :          * s : run size
     858                 :          * x : don't care
     859                 :          * - : 0
     860                 :          * [cdzla] : bit set
     861                 :          *
     862                 :          *   Unallocated:
     863                 :          *     ssssssss ssssssss ssss---- --c-----
     864                 :          *     xxxxxxxx xxxxxxxx xxxx---- ----d---
     865                 :          *     ssssssss ssssssss ssss---- -----z--
     866                 :          *
     867                 :          *   Small:
     868                 :          *     rrrrrrrr rrrrrrrr rrrr---- -------a
     869                 :          *     rrrrrrrr rrrrrrrr rrrr---- -------a
     870                 :          *     rrrrrrrr rrrrrrrr rrrr---- -------a
     871                 :          *
     872                 :          *   Large:
     873                 :          *     ssssssss ssssssss ssss---- ------la
     874                 :          *     -------- -------- -------- ------la
     875                 :          *     -------- -------- -------- ------la
     876                 :          */
     877                 :         size_t                          bits;
     878                 : 
     879                 : /* Note that CHUNK_MAP_DECOMMITTED's meaning varies depending on whether
     880                 :  * MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are defined.
     881                 :  *
     882                 :  * If MALLOC_DECOMMIT is defined, a page which is CHUNK_MAP_DECOMMITTED must be
     883                 :  * re-committed with pages_commit() before it may be touched.  If
     884                 :  * MALLOC_DECOMMIT is defined, MALLOC_DOUBLE_PURGE may not be defined.
     885                 :  *
     886                 :  * If neither MALLOC_DECOMMIT nor MALLOC_DOUBLE_PURGE is defined, pages which
     887                 :  * are madvised (with either MADV_DONTNEED or MADV_FREE) are marked with
     888                 :  * CHUNK_MAP_MADVISED.
     889                 :  *
     890                 :  * Otherwise, if MALLOC_DECOMMIT is not defined and MALLOC_DOUBLE_PURGE is
     891                 :  * defined, then a page which is madvised is marked as CHUNK_MAP_MADVISED.
     892                 :  * When it's finally freed with jemalloc_purge_freed_pages, the page is marked
     893                 :  * as CHUNK_MAP_DECOMMITTED.
     894                 :  */
     895                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
     896                 : #define CHUNK_MAP_MADVISED      ((size_t)0x40U)
     897                 : #define CHUNK_MAP_DECOMMITTED   ((size_t)0x20U)
     898                 : #define CHUNK_MAP_MADVISED_OR_DECOMMITTED (CHUNK_MAP_MADVISED | CHUNK_MAP_DECOMMITTED)
     899                 : #endif
     900                 : #define CHUNK_MAP_KEY           ((size_t)0x10U)
     901                 : #define CHUNK_MAP_DIRTY         ((size_t)0x08U)
     902                 : #define CHUNK_MAP_ZEROED        ((size_t)0x04U)
     903                 : #define CHUNK_MAP_LARGE         ((size_t)0x02U)
     904                 : #define CHUNK_MAP_ALLOCATED     ((size_t)0x01U)
     905                 : };
     906                 : typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
     907                 : typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
     908                 : 
     909                 : /* Arena chunk header. */
     910                 : typedef struct arena_chunk_s arena_chunk_t;
     911                 : struct arena_chunk_s {
     912                 :         /* Arena that owns the chunk. */
     913                 :         arena_t         *arena;
     914                 : 
     915                 :         /* Linkage for the arena's chunks_dirty tree. */
     916                 :         rb_node(arena_chunk_t) link_dirty;
     917                 : 
     918                 : #ifdef MALLOC_DOUBLE_PURGE
     919                 :         /* If we're double-purging, we maintain a linked list of chunks which
     920                 :          * have pages which have been madvise(MADV_FREE)'d but not explicitly
     921                 :          * purged.
     922                 :          *
     923                 :          * We're currently lazy and don't remove a chunk from this list when
     924                 :          * all its madvised pages are recommitted. */
     925                 :         LinkedList      chunks_madvised_elem;
     926                 : #endif
     927                 : 
     928                 :         /* Number of dirty pages. */
     929                 :         size_t          ndirty;
     930                 : 
     931                 :         /* Map of pages within chunk that keeps track of free/large/small. */
     932                 :         arena_chunk_map_t map[1]; /* Dynamically sized. */
     933                 : };
     934                 : typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
     935                 : 
     936                 : typedef struct arena_run_s arena_run_t;
     937                 : struct arena_run_s {
     938                 : #ifdef MALLOC_DEBUG
     939                 :         uint32_t        magic;
     940                 : #  define ARENA_RUN_MAGIC 0x384adf93
     941                 : #endif
     942                 : 
     943                 :         /* Bin this run is associated with. */
     944                 :         arena_bin_t     *bin;
     945                 : 
     946                 :         /* Index of first element that might have a free region. */
     947                 :         unsigned        regs_minelm;
     948                 : 
     949                 :         /* Number of free regions in run. */
     950                 :         unsigned        nfree;
     951                 : 
     952                 :         /* Bitmask of in-use regions (0: in use, 1: free). */
     953                 :         unsigned        regs_mask[1]; /* Dynamically sized. */
     954                 : };
     955                 : 
     956                 : struct arena_bin_s {
     957                 :         /*
     958                 :          * Current run being used to service allocations of this bin's size
     959                 :          * class.
     960                 :          */
     961                 :         arena_run_t     *runcur;
     962                 : 
     963                 :         /*
     964                 :          * Tree of non-full runs.  This tree is used when looking for an
     965                 :          * existing run when runcur is no longer usable.  We choose the
     966                 :          * non-full run that is lowest in memory; this policy tends to keep
     967                 :          * objects packed well, and it can also help reduce the number of
     968                 :          * almost-empty chunks.
     969                 :          */
     970                 :         arena_run_tree_t runs;
     971                 : 
     972                 :         /* Size of regions in a run for this bin's size class. */
     973                 :         size_t          reg_size;
     974                 : 
     975                 :         /* Total size of a run for this bin's size class. */
     976                 :         size_t          run_size;
     977                 : 
     978                 :         /* Total number of regions in a run for this bin's size class. */
     979                 :         uint32_t        nregs;
     980                 : 
     981                 :         /* Number of elements in a run's regs_mask for this bin's size class. */
     982                 :         uint32_t        regs_mask_nelms;
     983                 : 
     984                 :         /* Offset of first region in a run for this bin's size class. */
     985                 :         uint32_t        reg0_offset;
     986                 : 
     987                 : #ifdef MALLOC_STATS
     988                 :         /* Bin statistics. */
     989                 :         malloc_bin_stats_t stats;
     990                 : #endif
     991                 : };
     992                 : 
     993                 : struct arena_s {
     994                 : #ifdef MALLOC_DEBUG
     995                 :         uint32_t                magic;
     996                 : #  define ARENA_MAGIC 0x947d3d24
     997                 : #endif
     998                 : 
     999                 :         /* All operations on this arena require that lock be locked. */
    1000                 : #ifdef MOZ_MEMORY
    1001                 :         malloc_spinlock_t       lock;
    1002                 : #else
    1003                 :         pthread_mutex_t         lock;
    1004                 : #endif
    1005                 : 
    1006                 : #ifdef MALLOC_STATS
    1007                 :         arena_stats_t           stats;
    1008                 : #endif
    1009                 : 
    1010                 :         /* Tree of dirty-page-containing chunks this arena manages. */
    1011                 :         arena_chunk_tree_t      chunks_dirty;
    1012                 : 
    1013                 : #ifdef MALLOC_DOUBLE_PURGE
    1014                 :         /* Head of a linked list of MADV_FREE'd-page-containing chunks this
    1015                 :          * arena manages. */
    1016                 :         LinkedList              chunks_madvised;
    1017                 : #endif
    1018                 : 
    1019                 :         /*
    1020                 :          * In order to avoid rapid chunk allocation/deallocation when an arena
    1021                 :          * oscillates right on the cusp of needing a new chunk, cache the most
    1022                 :          * recently freed chunk.  The spare is left in the arena's chunk trees
    1023                 :          * until it is deleted.
    1024                 :          *
    1025                 :          * There is one spare chunk per arena, rather than one spare total, in
    1026                 :          * order to avoid interactions between multiple threads that could make
    1027                 :          * a single spare inadequate.
    1028                 :          */
    1029                 :         arena_chunk_t           *spare;
    1030                 : 
    1031                 :         /*
    1032                 :          * Current count of pages within unused runs that are potentially
    1033                 :          * dirty, and for which madvise(... MADV_FREE) has not been called.  By
    1034                 :          * tracking this, we can institute a limit on how much dirty unused
    1035                 :          * memory is mapped for each arena.
    1036                 :          */
    1037                 :         size_t                  ndirty;
    1038                 : 
    1039                 :         /*
    1040                 :          * Size/address-ordered tree of this arena's available runs.  This tree
    1041                 :          * is used for first-best-fit run allocation.
    1042                 :          */
    1043                 :         arena_avail_tree_t      runs_avail;
    1044                 : 
    1045                 : #ifdef MALLOC_BALANCE
    1046                 :         /*
    1047                 :          * The arena load balancing machinery needs to keep track of how much
    1048                 :          * lock contention there is.  This value is exponentially averaged.
    1049                 :          */
    1050                 :         uint32_t                contention;
    1051                 : #endif
    1052                 : 
    1053                 :         /*
    1054                 :          * bins is used to store rings of free regions of the following sizes,
    1055                 :          * assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS.
    1056                 :          *
    1057                 :          *   bins[i] | size |
    1058                 :          *   --------+------+
    1059                 :          *        0  |    2 |
    1060                 :          *        1  |    4 |
    1061                 :          *        2  |    8 |
    1062                 :          *   --------+------+
    1063                 :          *        3  |   16 |
    1064                 :          *        4  |   32 |
    1065                 :          *        5  |   48 |
    1066                 :          *        6  |   64 |
    1067                 :          *           :      :
    1068                 :          *           :      :
    1069                 :          *       33  |  496 |
    1070                 :          *       34  |  512 |
    1071                 :          *   --------+------+
    1072                 :          *       35  | 1024 |
    1073                 :          *       36  | 2048 |
    1074                 :          *   --------+------+
    1075                 :          */
    1076                 :         arena_bin_t             bins[1]; /* Dynamically sized. */
    1077                 : };
    1078                 : 
    1079                 : /******************************************************************************/
    1080                 : /*
    1081                 :  * Data.
    1082                 :  */
    1083                 : 
    1084                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    1085                 : /* Number of CPUs. */
    1086                 : static unsigned         ncpus;
    1087                 : #endif
    1088                 : 
    1089                 : /*
    1090                 :  * When MALLOC_STATIC_SIZES is defined most of the parameters
    1091                 :  * controlling the malloc behavior are defined as compile-time constants
    1092                 :  * for best performance and cannot be altered at runtime.
    1093                 :  */
    1094                 : #define MALLOC_STATIC_SIZES 1
    1095                 : 
    1096                 : #ifdef MALLOC_STATIC_SIZES
    1097                 : 
    1098                 : /*
    1099                 :  * VM page size. It must divide the runtime CPU page size or the code
    1100                 :  * will abort.
    1101                 :  */
    1102                 : #define pagesize_2pow                   ((size_t) 12)
    1103                 : #define pagesize                        ((size_t) 1 << pagesize_2pow)
    1104                 : #define pagesize_mask                   (pagesize - 1)
    1105                 : 
    1106                 : /* Various quantum-related settings. */
    1107                 : 
    1108                 : #define QUANTUM_DEFAULT                 ((size_t) 1 << QUANTUM_2POW_MIN)
    1109                 : static const size_t     quantum =       QUANTUM_DEFAULT;
    1110                 : static const size_t     quantum_mask =  QUANTUM_DEFAULT - 1;
    1111                 : 
    1112                 : /* Various bin-related settings. */
    1113                 : 
    1114                 : static const size_t     small_min =     (QUANTUM_DEFAULT >> 1) + 1;
    1115                 : static const size_t     small_max =     (size_t) SMALL_MAX_DEFAULT;
    1116                 : 
    1117                 : /* Max size class for bins. */
    1118                 : static const size_t     bin_maxclass =  pagesize >> 1;
    1119                 : 
    1120                 :  /* Number of (2^n)-spaced tiny bins. */
    1121                 : static const unsigned   ntbins =        (unsigned)
    1122                 :                                         (QUANTUM_2POW_MIN - TINY_MIN_2POW);
    1123                 : 
    1124                 :  /* Number of quantum-spaced bins. */
    1125                 : static const unsigned   nqbins =        (unsigned)
    1126                 :                                         (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN);
    1127                 : 
    1128                 : /* Number of (2^n)-spaced sub-page bins. */
    1129                 : static const unsigned   nsbins =        (unsigned)
    1130                 :                                         (pagesize_2pow -
    1131                 :                                          SMALL_MAX_2POW_DEFAULT - 1);
    1132                 : 
    1133                 : #else /* !MALLOC_STATIC_SIZES */
    1134                 : 
    1135                 : /* VM page size. */
    1136                 : static size_t           pagesize;
    1137                 : static size_t           pagesize_mask;
    1138                 : static size_t           pagesize_2pow;
    1139                 : 
    1140                 : /* Various bin-related settings. */
    1141                 : static size_t           bin_maxclass; /* Max size class for bins. */
    1142                 : static unsigned         ntbins; /* Number of (2^n)-spaced tiny bins. */
    1143                 : static unsigned         nqbins; /* Number of quantum-spaced bins. */
    1144                 : static unsigned         nsbins; /* Number of (2^n)-spaced sub-page bins. */
    1145                 : static size_t           small_min;
    1146                 : static size_t           small_max;
    1147                 : 
    1148                 : /* Various quantum-related settings. */
    1149                 : static size_t           quantum;
    1150                 : static size_t           quantum_mask; /* (quantum - 1). */
    1151                 : 
    1152                 : #endif
    1153                 : 
    1154                 : /* Various chunk-related settings. */
    1155                 : 
    1156                 : /*
    1157                 :  * Compute the header size such that it is large enough to contain the page map
    1158                 :  * and enough nodes for the worst case: one node per non-header page plus one
    1159                 :  * extra for situations where we briefly have one more node allocated than we
    1160                 :  * will need.
    1161                 :  */
    1162                 : #define calculate_arena_header_size()                                   \
    1163                 :         (sizeof(arena_chunk_t) + sizeof(arena_chunk_map_t) * (chunk_npages - 1))
    1164                 : 
    1165                 : #define calculate_arena_header_pages()                                  \
    1166                 :         ((calculate_arena_header_size() >> pagesize_2pow) +               \
    1167                 :          ((calculate_arena_header_size() & pagesize_mask) ? 1 : 0))
    1168                 : 
    1169                 : /* Max size class for arenas. */
    1170                 : #define calculate_arena_maxclass()                                      \
    1171                 :         (chunksize - (arena_chunk_header_npages << pagesize_2pow))
    1172                 : 
    1173                 : #ifdef MALLOC_STATIC_SIZES
    1174                 : #define CHUNKSIZE_DEFAULT               ((size_t) 1 << CHUNK_2POW_DEFAULT)
    1175                 : static const size_t     chunksize =     CHUNKSIZE_DEFAULT;
    1176                 : static const size_t     chunksize_mask =CHUNKSIZE_DEFAULT - 1;
    1177                 : static const size_t     chunk_npages =  CHUNKSIZE_DEFAULT >> pagesize_2pow;
    1178                 : #define arena_chunk_header_npages       calculate_arena_header_pages()
    1179                 : #define arena_maxclass                  calculate_arena_maxclass()
    1180                 : #else
    1181                 : static size_t           chunksize;
    1182                 : static size_t           chunksize_mask; /* (chunksize - 1). */
    1183                 : static size_t           chunk_npages;
    1184                 : static size_t           arena_chunk_header_npages;
    1185                 : static size_t           arena_maxclass; /* Max size class for arenas. */
    1186                 : #endif
    1187                 : 
    1188                 : /********/
    1189                 : /*
    1190                 :  * Chunks.
    1191                 :  */
    1192                 : 
    1193                 : #ifdef MALLOC_VALIDATE
    1194                 : static malloc_rtree_t *chunk_rtree;
    1195                 : #endif
    1196                 : 
    1197                 : /* Protects chunk-related data structures. */
    1198                 : static malloc_mutex_t   huge_mtx;
    1199                 : 
    1200                 : /* Tree of chunks that are stand-alone huge allocations. */
    1201                 : static extent_tree_t    huge;
    1202                 : 
    1203                 : #ifdef MALLOC_STATS
    1204                 : /* Huge allocation statistics. */
    1205                 : static uint64_t         huge_nmalloc;
    1206                 : static uint64_t         huge_ndalloc;
    1207                 : static size_t           huge_allocated;
    1208                 : #endif
    1209                 : 
    1210                 : #ifdef MALLOC_PAGEFILE
    1211                 : static char             pagefile_templ[PATH_MAX];
    1212                 : #endif
    1213                 : 
    1214                 : /****************************/
    1215                 : /*
    1216                 :  * base (internal allocation).
    1217                 :  */
    1218                 : 
    1219                 : /*
    1220                 :  * Current pages that are being used for internal memory allocations.  These
    1221                 :  * pages are carved up in cacheline-size quanta, so that there is no chance of
    1222                 :  * false cache line sharing.
    1223                 :  */
    1224                 : static void             *base_pages;
    1225                 : static void             *base_next_addr;
    1226                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
    1227                 : static void             *base_next_decommitted;
    1228                 : #endif
    1229                 : static void             *base_past_addr; /* Addr immediately past base_pages. */
    1230                 : static extent_node_t    *base_nodes;
    1231                 : static malloc_mutex_t   base_mtx;
    1232                 : #ifdef MALLOC_STATS
    1233                 : static size_t           base_mapped;
    1234                 : static size_t           base_committed;
    1235                 : #endif
    1236                 : 
    1237                 : /********/
    1238                 : /*
    1239                 :  * Arenas.
    1240                 :  */
    1241                 : 
    1242                 : /*
    1243                 :  * Arenas that are used to service external requests.  Not all elements of the
    1244                 :  * arenas array are necessarily used; arenas are created lazily as needed.
    1245                 :  */
    1246                 : static arena_t          **arenas;
    1247                 : static unsigned         narenas;
    1248                 : #ifndef NO_TLS
    1249                 : #  ifdef MALLOC_BALANCE
    1250                 : static unsigned         narenas_2pow;
    1251                 : #  else
    1252                 : static unsigned         next_arena;
    1253                 : #  endif
    1254                 : #endif
    1255                 : #ifdef MOZ_MEMORY
    1256                 : static malloc_spinlock_t arenas_lock; /* Protects arenas initialization. */
    1257                 : #else
    1258                 : static pthread_mutex_t arenas_lock; /* Protects arenas initialization. */
    1259                 : #endif
    1260                 : 
    1261                 : #ifndef NO_TLS
    1262                 : /*
    1263                 :  * Map of pthread_self() --> arenas[???], used for selecting an arena to use
    1264                 :  * for allocations.
    1265                 :  */
    1266                 : #ifndef MOZ_MEMORY_WINDOWS
    1267                 : static __thread arena_t *arenas_map;
    1268                 : #endif
    1269                 : #endif
    1270                 : 
    1271                 : #ifdef MALLOC_STATS
    1272                 : /* Chunk statistics. */
    1273                 : static chunk_stats_t    stats_chunks;
    1274                 : #endif
    1275                 : 
    1276                 : /*******************************/
    1277                 : /*
    1278                 :  * Runtime configuration options.
    1279                 :  */
    1280                 : const char      *_malloc_options;
    1281                 : 
    1282                 : #ifndef MALLOC_PRODUCTION
    1283                 : static bool     opt_abort = true;
    1284                 : #ifdef MALLOC_FILL
    1285                 : static bool     opt_junk = true;
    1286                 : #endif
    1287                 : #else
    1288                 : static bool     opt_abort = false;
    1289                 : #ifdef MALLOC_FILL
    1290                 : static bool     opt_junk = false;
    1291                 : #endif
    1292                 : #endif
    1293                 : static size_t   opt_dirty_max = DIRTY_MAX_DEFAULT;
    1294                 : #ifdef MALLOC_BALANCE
    1295                 : static uint64_t opt_balance_threshold = BALANCE_THRESHOLD_DEFAULT;
    1296                 : #endif
    1297                 : static bool     opt_print_stats = false;
    1298                 : #ifdef MALLOC_STATIC_SIZES
    1299                 : #define opt_quantum_2pow        QUANTUM_2POW_MIN
    1300                 : #define opt_small_max_2pow      SMALL_MAX_2POW_DEFAULT
    1301                 : #define opt_chunk_2pow          CHUNK_2POW_DEFAULT
    1302                 : #else
    1303                 : static size_t   opt_quantum_2pow = QUANTUM_2POW_MIN;
    1304                 : static size_t   opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT;
    1305                 : static size_t   opt_chunk_2pow = CHUNK_2POW_DEFAULT;
    1306                 : #endif
    1307                 : #ifdef MALLOC_PAGEFILE
    1308                 : static bool     opt_pagefile = false;
    1309                 : #endif
    1310                 : #ifdef MALLOC_UTRACE
    1311                 : static bool     opt_utrace = false;
    1312                 : #endif
    1313                 : #ifdef MALLOC_SYSV
    1314                 : static bool     opt_sysv = false;
    1315                 : #endif
    1316                 : #ifdef MALLOC_XMALLOC
    1317                 : static bool     opt_xmalloc = false;
    1318                 : #endif
    1319                 : #ifdef MALLOC_FILL
    1320                 : static bool     opt_zero = false;
    1321                 : #endif
    1322                 : static int      opt_narenas_lshift = 0;
    1323                 : 
    1324                 : #ifdef MALLOC_UTRACE
    1325                 : typedef struct {
    1326                 :         void    *p;
    1327                 :         size_t  s;
    1328                 :         void    *r;
    1329                 : } malloc_utrace_t;
    1330                 : 
    1331                 : #define UTRACE(a, b, c)                                                 \
    1332                 :         if (opt_utrace) {                                               \
    1333                 :                 malloc_utrace_t ut;                                     \
    1334                 :                 ut.p = (a);                                             \
    1335                 :                 ut.s = (b);                                             \
    1336                 :                 ut.r = (c);                                             \
    1337                 :                 utrace(&ut, sizeof(ut));                            \
    1338                 :         }
    1339                 : #else
    1340                 : #define UTRACE(a, b, c)
    1341                 : #endif
    1342                 : 
    1343                 : /******************************************************************************/
    1344                 : /*
    1345                 :  * Begin function prototypes for non-inline static functions.
    1346                 :  */
    1347                 : 
    1348                 : static char     *umax2s(uintmax_t x, unsigned base, char *s);
    1349                 : static bool     malloc_mutex_init(malloc_mutex_t *mutex);
    1350                 : static bool     malloc_spin_init(malloc_spinlock_t *lock);
    1351                 : static void     wrtmessage(const char *p1, const char *p2, const char *p3,
    1352                 :                 const char *p4);
    1353                 : #ifdef MALLOC_STATS
    1354                 : #ifdef MOZ_MEMORY_DARWIN
    1355                 : /* Avoid namespace collision with OS X's malloc APIs. */
    1356                 : #define malloc_printf moz_malloc_printf
    1357                 : #endif
    1358                 : static void     malloc_printf(const char *format, ...);
    1359                 : #endif
    1360                 : static bool     base_pages_alloc_mmap(size_t minsize);
    1361                 : static bool     base_pages_alloc(size_t minsize);
    1362                 : static void     *base_alloc(size_t size);
    1363                 : static void     *base_calloc(size_t number, size_t size);
    1364                 : static extent_node_t *base_node_alloc(void);
    1365                 : static void     base_node_dealloc(extent_node_t *node);
    1366                 : #ifdef MALLOC_STATS
    1367                 : static void     stats_print(arena_t *arena);
    1368                 : #endif
    1369                 : static void     *pages_map(void *addr, size_t size, int pfd);
    1370                 : static void     pages_unmap(void *addr, size_t size);
    1371                 : static void     *chunk_alloc_mmap(size_t size, bool pagefile);
    1372                 : #ifdef MALLOC_PAGEFILE
    1373                 : static int      pagefile_init(size_t size);
    1374                 : static void     pagefile_close(int pfd);
    1375                 : #endif
    1376                 : static void     *chunk_alloc(size_t size, bool zero, bool pagefile);
    1377                 : static void     chunk_dealloc_mmap(void *chunk, size_t size);
    1378                 : static void     chunk_dealloc(void *chunk, size_t size);
    1379                 : #ifndef NO_TLS
    1380                 : static arena_t  *choose_arena_hard(void);
    1381                 : #endif
    1382                 : static void     arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
    1383                 :     bool large, bool zero);
    1384                 : static void arena_chunk_init(arena_t *arena, arena_chunk_t *chunk);
    1385                 : static void     arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk);
    1386                 : static arena_run_t *arena_run_alloc(arena_t *arena, arena_bin_t *bin,
    1387                 :     size_t size, bool large, bool zero);
    1388                 : static void     arena_purge(arena_t *arena);
    1389                 : static void     arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
    1390                 : static void     arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
    1391                 :     arena_run_t *run, size_t oldsize, size_t newsize);
    1392                 : static void     arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
    1393                 :     arena_run_t *run, size_t oldsize, size_t newsize, bool dirty);
    1394                 : static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
    1395                 : static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
    1396                 : static size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size);
    1397                 : #ifdef MALLOC_BALANCE
    1398                 : static void     arena_lock_balance_hard(arena_t *arena);
    1399                 : #endif
    1400                 : static void     *arena_malloc_large(arena_t *arena, size_t size, bool zero);
    1401                 : static void     *arena_palloc(arena_t *arena, size_t alignment, size_t size,
    1402                 :     size_t alloc_size);
    1403                 : static size_t   arena_salloc(const void *ptr);
    1404                 : static void     arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk,
    1405                 :     void *ptr);
    1406                 : static void     arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk,
    1407                 :     void *ptr, size_t size, size_t oldsize);
    1408                 : static bool     arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk,
    1409                 :     void *ptr, size_t size, size_t oldsize);
    1410                 : static bool     arena_ralloc_large(void *ptr, size_t size, size_t oldsize);
    1411                 : static void     *arena_ralloc(void *ptr, size_t size, size_t oldsize);
    1412                 : static bool     arena_new(arena_t *arena);
    1413                 : static arena_t  *arenas_extend(unsigned ind);
    1414                 : static void     *huge_malloc(size_t size, bool zero);
    1415                 : static void     *huge_palloc(size_t alignment, size_t size);
    1416                 : static void     *huge_ralloc(void *ptr, size_t size, size_t oldsize);
    1417                 : static void     huge_dalloc(void *ptr);
    1418                 : static void     malloc_print_stats(void);
    1419                 : #ifndef MOZ_MEMORY_WINDOWS
    1420                 : static
    1421                 : #endif
    1422                 : bool            malloc_init_hard(void);
    1423                 : 
    1424                 : static void     _malloc_prefork(void);
    1425                 : static void     _malloc_postfork(void);
    1426                 : 
    1427                 : #ifdef MOZ_MEMORY_DARWIN
    1428                 : /*
    1429                 :  * MALLOC_ZONE_T_NOTE
    1430                 :  *
    1431                 :  * On Darwin, we hook into the memory allocator using a malloc_zone_t struct.
    1432                 :  * We must be very careful around this struct because of different behaviour on
    1433                 :  * different versions of OSX.
    1434                 :  *
    1435                 :  * Each of OSX 10.5, 10.6 and 10.7 use different versions of the struct
    1436                 :  * (with version numbers 3, 6 and 8 respectively). The binary we use on each of
    1437                 :  * these platforms will not necessarily be built using the correct SDK [1].
    1438                 :  * This means we need to statically know the correct struct size to use on all
    1439                 :  * OSX releases, and have a fallback for unknown future versions. The struct
    1440                 :  * sizes defined in osx_zone_types.h.
    1441                 :  *
    1442                 :  * For OSX 10.8 and later, we may expect the malloc_zone_t struct to change
    1443                 :  * again, and need to dynamically account for this. By simply leaving
    1444                 :  * malloc_zone_t alone, we don't quite deal with the problem, because there
    1445                 :  * remain calls to jemalloc through the mozalloc interface. We check this
    1446                 :  * dynamically on each allocation, using the CHECK_DARWIN macro and
    1447                 :  * osx_use_jemalloc.
    1448                 :  *
    1449                 :  *
    1450                 :  * [1] Mozilla is built as a universal binary on Mac, supporting i386 and
    1451                 :  *     x86_64. The i386 target is built using the 10.5 SDK, even if it runs on
    1452                 :  *     10.6. The x86_64 target is built using the 10.6 SDK, even if it runs on
    1453                 :  *     10.7 or later, or 10.5.
    1454                 :  *
    1455                 :  * FIXME:
    1456                 :  *   When later versions of OSX come out (10.8 and up), we need to check their
    1457                 :  *   malloc_zone_t versions. If they're greater than 8, we need a new version
    1458                 :  *   of malloc_zone_t adapted into osx_zone_types.h.
    1459                 :  */
    1460                 : 
    1461                 : #include "osx_zone_types.h"
    1462                 : 
    1463                 : #define LEOPARD_MALLOC_ZONE_T_VERSION 3
    1464                 : #define SNOW_LEOPARD_MALLOC_ZONE_T_VERSION 6
    1465                 : #define LION_MALLOC_ZONE_T_VERSION 8
    1466                 : 
    1467                 : static bool osx_use_jemalloc = false;
    1468                 : 
    1469                 : 
    1470                 : static lion_malloc_zone l_szone;
    1471                 : static malloc_zone_t * szone = (malloc_zone_t*)(&l_szone);
    1472                 : 
    1473                 : static lion_malloc_introspection l_ozone_introspect;
    1474                 : static malloc_introspection_t * const ozone_introspect =
    1475                 :         (malloc_introspection_t*)(&l_ozone_introspect);
    1476                 : static void szone2ozone(malloc_zone_t *zone, size_t size);
    1477                 : static size_t zone_version_size(int version);
    1478                 : #endif
    1479                 : 
    1480                 : /*
    1481                 :  * End function prototypes.
    1482                 :  */
    1483                 : /******************************************************************************/
    1484                 : 
    1485                 : /*
    1486                 :  * umax2s() provides minimal integer printing functionality, which is
    1487                 :  * especially useful for situations where allocation in vsnprintf() calls would
    1488                 :  * potentially cause deadlock.
    1489                 :  */
    1490                 : #define UMAX2S_BUFSIZE  65
    1491                 : char *
    1492               0 : umax2s(uintmax_t x, unsigned base, char *s)
    1493                 : {
    1494                 :         unsigned i;
    1495                 : 
    1496               0 :         i = UMAX2S_BUFSIZE - 1;
    1497               0 :         s[i] = '\0';
    1498               0 :         switch (base) {
    1499                 :         case 10:
    1500                 :                 do {
    1501               0 :                         i--;
    1502               0 :                         s[i] = "0123456789"[x % 10];
    1503               0 :                         x /= 10;
    1504               0 :                 } while (x > 0);
    1505               0 :                 break;
    1506                 :         case 16:
    1507                 :                 do {
    1508               0 :                         i--;
    1509               0 :                         s[i] = "0123456789abcdef"[x & 0xf];
    1510               0 :                         x >>= 4;
    1511               0 :                 } while (x > 0);
    1512               0 :                 break;
    1513                 :         default:
    1514                 :                 do {
    1515               0 :                         i--;
    1516               0 :                         s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % base];
    1517               0 :                         x /= base;
    1518               0 :                 } while (x > 0);
    1519                 :         }
    1520                 : 
    1521               0 :         return (&s[i]);
    1522                 : }
    1523                 : 
    1524                 : static void
    1525               0 : wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
    1526                 : {
    1527                 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_WINDOWS)
    1528                 : #define _write  write
    1529                 : #endif
    1530               0 :         _write(STDERR_FILENO, p1, (unsigned int) strlen(p1));
    1531               0 :         _write(STDERR_FILENO, p2, (unsigned int) strlen(p2));
    1532               0 :         _write(STDERR_FILENO, p3, (unsigned int) strlen(p3));
    1533               0 :         _write(STDERR_FILENO, p4, (unsigned int) strlen(p4));
    1534               0 : }
    1535                 : 
    1536                 : #define _malloc_message malloc_message
    1537                 : 
    1538                 : void    (*_malloc_message)(const char *p1, const char *p2, const char *p3,
    1539                 :             const char *p4) = wrtmessage;
    1540                 : 
    1541                 : #ifdef MALLOC_DEBUG
    1542                 : #  define assert(e) do {                                                \
    1543                 :         if (!(e)) {                                                     \
    1544                 :                 char line_buf[UMAX2S_BUFSIZE];                          \
    1545                 :                 _malloc_message(__FILE__, ":", umax2s(__LINE__, 10,   \
    1546                 :                     line_buf), ": Failed assertion: ");                       \
    1547                 :                 _malloc_message("\"", #e, "\"\n", "");                  \
    1548                 :                 abort();                                                \
    1549                 :         }                                                               \
    1550                 : } while (0)
    1551                 : #else
    1552                 : #define assert(e)
    1553                 : #endif
    1554                 : 
    1555                 : /******************************************************************************/
    1556                 : /*
    1557                 :  * Begin mutex.  We can't use normal pthread mutexes in all places, because
    1558                 :  * they require malloc()ed memory, which causes bootstrapping issues in some
    1559                 :  * cases.
    1560                 :  */
    1561                 : 
    1562                 : static bool
    1563           93345 : malloc_mutex_init(malloc_mutex_t *mutex)
    1564                 : {
    1565                 : #if defined(MOZ_MEMORY_WINDOWS)
    1566                 :         if (isthreaded)
    1567                 :                 if (! __crtInitCritSecAndSpinCount(mutex, _CRT_SPINCOUNT))
    1568                 :                         return (true);
    1569                 : #elif defined(MOZ_MEMORY_DARWIN)
    1570                 :         mutex->lock = OS_SPINLOCK_INIT;
    1571                 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
    1572                 :         pthread_mutexattr_t attr;
    1573           93345 :         if (pthread_mutexattr_init(&attr) != 0)
    1574               0 :                 return (true);
    1575           93345 :         pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
    1576           93345 :         if (pthread_mutex_init(mutex, &attr) != 0) {
    1577               0 :                 pthread_mutexattr_destroy(&attr);
    1578               0 :                 return (true);
    1579                 :         }
    1580           93345 :         pthread_mutexattr_destroy(&attr);
    1581                 : #elif defined(MOZ_MEMORY)
    1582                 :         if (pthread_mutex_init(mutex, NULL) != 0)
    1583                 :                 return (true);
    1584                 : #else
    1585                 :         static const spinlock_t lock = _SPINLOCK_INITIALIZER;
    1586                 : 
    1587                 :         mutex->lock = lock;
    1588                 : #endif
    1589           93345 :         return (false);
    1590                 : }
    1591                 : 
    1592                 : static inline void
    1593        90845182 : malloc_mutex_lock(malloc_mutex_t *mutex)
    1594                 : {
    1595                 : 
    1596                 : #if defined(MOZ_MEMORY_WINDOWS)
    1597                 :         EnterCriticalSection(mutex);
    1598                 : #elif defined(MOZ_MEMORY_DARWIN)
    1599                 :         OSSpinLockLock(&mutex->lock);
    1600                 : #elif defined(MOZ_MEMORY)
    1601        90845182 :         pthread_mutex_lock(mutex);
    1602                 : #else
    1603                 :         if (isthreaded)
    1604                 :                 _SPINLOCK(&mutex->lock);
    1605                 : #endif
    1606        90845224 : }
    1607                 : 
    1608                 : static inline void
    1609        90845224 : malloc_mutex_unlock(malloc_mutex_t *mutex)
    1610                 : {
    1611                 : 
    1612                 : #if defined(MOZ_MEMORY_WINDOWS)
    1613                 :         LeaveCriticalSection(mutex);
    1614                 : #elif defined(MOZ_MEMORY_DARWIN)
    1615                 :         OSSpinLockUnlock(&mutex->lock);
    1616                 : #elif defined(MOZ_MEMORY)
    1617        90845224 :         pthread_mutex_unlock(mutex);
    1618                 : #else
    1619                 :         if (isthreaded)
    1620                 :                 _SPINUNLOCK(&mutex->lock);
    1621                 : #endif
    1622        90845222 : }
    1623                 : 
    1624                 : static bool
    1625               0 : malloc_spin_init(malloc_spinlock_t *lock)
    1626                 : {
    1627                 : #if defined(MOZ_MEMORY_WINDOWS)
    1628                 :         if (isthreaded)
    1629                 :                 if (! __crtInitCritSecAndSpinCount(lock, _CRT_SPINCOUNT))
    1630                 :                         return (true);
    1631                 : #elif defined(MOZ_MEMORY_DARWIN)
    1632                 :         lock->lock = OS_SPINLOCK_INIT;
    1633                 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
    1634                 :         pthread_mutexattr_t attr;
    1635               0 :         if (pthread_mutexattr_init(&attr) != 0)
    1636               0 :                 return (true);
    1637               0 :         pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
    1638               0 :         if (pthread_mutex_init(lock, &attr) != 0) {
    1639               0 :                 pthread_mutexattr_destroy(&attr);
    1640               0 :                 return (true);
    1641                 :         }
    1642               0 :         pthread_mutexattr_destroy(&attr);
    1643                 : #elif defined(MOZ_MEMORY)
    1644                 :         if (pthread_mutex_init(lock, NULL) != 0)
    1645                 :                 return (true);
    1646                 : #else
    1647                 :         lock->lock = _SPINLOCK_INITIALIZER;
    1648                 : #endif
    1649               0 :         return (false);
    1650                 : }
    1651                 : 
    1652                 : static inline void
    1653               0 : malloc_spin_lock(malloc_spinlock_t *lock)
    1654                 : {
    1655                 : 
    1656                 : #if defined(MOZ_MEMORY_WINDOWS)
    1657                 :         EnterCriticalSection(lock);
    1658                 : #elif defined(MOZ_MEMORY_DARWIN)
    1659                 :         OSSpinLockLock(&lock->lock);
    1660                 : #elif defined(MOZ_MEMORY)
    1661               0 :         pthread_mutex_lock(lock);
    1662                 : #else
    1663                 :         if (isthreaded)
    1664                 :                 _SPINLOCK(&lock->lock);
    1665                 : #endif
    1666               0 : }
    1667                 : 
    1668                 : static inline void
    1669               0 : malloc_spin_unlock(malloc_spinlock_t *lock)
    1670                 : {
    1671                 : #if defined(MOZ_MEMORY_WINDOWS)
    1672                 :         LeaveCriticalSection(lock);
    1673                 : #elif defined(MOZ_MEMORY_DARWIN)
    1674                 :         OSSpinLockUnlock(&lock->lock);
    1675                 : #elif defined(MOZ_MEMORY)
    1676               0 :         pthread_mutex_unlock(lock);
    1677                 : #else
    1678                 :         if (isthreaded)
    1679                 :                 _SPINUNLOCK(&lock->lock);
    1680                 : #endif
    1681               0 : }
    1682                 : 
    1683                 : /*
    1684                 :  * End mutex.
    1685                 :  */
    1686                 : /******************************************************************************/
    1687                 : /*
    1688                 :  * Begin spin lock.  Spin locks here are actually adaptive mutexes that block
    1689                 :  * after a period of spinning, because unbounded spinning would allow for
    1690                 :  * priority inversion.
    1691                 :  */
    1692                 : 
    1693                 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_DARWIN)
    1694                 : #  define       malloc_spin_init        malloc_mutex_init
    1695                 : #  define       malloc_spin_lock        malloc_mutex_lock
    1696                 : #  define       malloc_spin_unlock      malloc_mutex_unlock
    1697                 : #endif
    1698                 : 
    1699                 : #ifndef MOZ_MEMORY
    1700                 : /*
    1701                 :  * We use an unpublished interface to initialize pthread mutexes with an
    1702                 :  * allocation callback, in order to avoid infinite recursion.
    1703                 :  */
    1704                 : int     _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
    1705                 :     void *(calloc_cb)(size_t, size_t));
    1706                 : 
    1707                 : __weak_reference(_pthread_mutex_init_calloc_cb_stub,
    1708                 :     _pthread_mutex_init_calloc_cb);
    1709                 : 
    1710                 : int
    1711                 : _pthread_mutex_init_calloc_cb_stub(pthread_mutex_t *mutex,
    1712                 :     void *(calloc_cb)(size_t, size_t))
    1713                 : {
    1714                 : 
    1715                 :         return (0);
    1716                 : }
    1717                 : 
    1718                 : static bool
    1719                 : malloc_spin_init(pthread_mutex_t *lock)
    1720                 : {
    1721                 : 
    1722                 :         if (_pthread_mutex_init_calloc_cb(lock, base_calloc) != 0)
    1723                 :                 return (true);
    1724                 : 
    1725                 :         return (false);
    1726                 : }
    1727                 : 
    1728                 : static inline unsigned
    1729                 : malloc_spin_lock(pthread_mutex_t *lock)
    1730                 : {
    1731                 :         unsigned ret = 0;
    1732                 : 
    1733                 :         if (isthreaded) {
    1734                 :                 if (_pthread_mutex_trylock(lock) != 0) {
    1735                 :                         unsigned i;
    1736                 :                         volatile unsigned j;
    1737                 : 
    1738                 :                         /* Exponentially back off. */
    1739                 :                         for (i = 1; i <= SPIN_LIMIT_2POW; i++) {
    1740                 :                                 for (j = 0; j < (1U << i); j++)
    1741                 :                                         ret++;
    1742                 : 
    1743                 :                                 CPU_SPINWAIT;
    1744                 :                                 if (_pthread_mutex_trylock(lock) == 0)
    1745                 :                                         return (ret);
    1746                 :                         }
    1747                 : 
    1748                 :                         /*
    1749                 :                          * Spinning failed.  Block until the lock becomes
    1750                 :                          * available, in order to avoid indefinite priority
    1751                 :                          * inversion.
    1752                 :                          */
    1753                 :                         _pthread_mutex_lock(lock);
    1754                 :                         assert((ret << BLOCK_COST_2POW) != 0);
    1755                 :                         return (ret << BLOCK_COST_2POW);
    1756                 :                 }
    1757                 :         }
    1758                 : 
    1759                 :         return (ret);
    1760                 : }
    1761                 : 
    1762                 : static inline void
    1763                 : malloc_spin_unlock(pthread_mutex_t *lock)
    1764                 : {
    1765                 : 
    1766                 :         if (isthreaded)
    1767                 :                 _pthread_mutex_unlock(lock);
    1768                 : }
    1769                 : #endif
    1770                 : 
    1771                 : /*
    1772                 :  * End spin lock.
    1773                 :  */
    1774                 : /******************************************************************************/
    1775                 : /*
    1776                 :  * Begin Utility functions/macros.
    1777                 :  */
    1778                 : 
    1779                 : /* Return the chunk address for allocation address a. */
    1780                 : #define CHUNK_ADDR2BASE(a)                                              \
    1781                 :         ((void *)((uintptr_t)(a) & ~chunksize_mask))
    1782                 : 
    1783                 : /* Return the chunk offset of address a. */
    1784                 : #define CHUNK_ADDR2OFFSET(a)                                            \
    1785                 :         ((size_t)((uintptr_t)(a) & chunksize_mask))
    1786                 : 
    1787                 : /* Return the smallest chunk multiple that is >= s. */
    1788                 : #define CHUNK_CEILING(s)                                                \
    1789                 :         (((s) + chunksize_mask) & ~chunksize_mask)
    1790                 : 
    1791                 : /* Return the smallest cacheline multiple that is >= s. */
    1792                 : #define CACHELINE_CEILING(s)                                            \
    1793                 :         (((s) + (CACHELINE - 1)) & ~(CACHELINE - 1))
    1794                 : 
    1795                 : /* Return the smallest quantum multiple that is >= a. */
    1796                 : #define QUANTUM_CEILING(a)                                              \
    1797                 :         (((a) + quantum_mask) & ~quantum_mask)
    1798                 : 
    1799                 : /* Return the smallest pagesize multiple that is >= s. */
    1800                 : #define PAGE_CEILING(s)                                                 \
    1801                 :         (((s) + pagesize_mask) & ~pagesize_mask)
    1802                 : 
    1803                 : /* Compute the smallest power of 2 that is >= x. */
    1804                 : static inline size_t
    1805        14496575 : pow2_ceil(size_t x)
    1806                 : {
    1807                 : 
    1808        14496575 :         x--;
    1809        14496575 :         x |= x >> 1;
    1810        14496575 :         x |= x >> 2;
    1811        14496575 :         x |= x >> 4;
    1812        14496575 :         x |= x >> 8;
    1813        14496575 :         x |= x >> 16;
    1814                 : #if (SIZEOF_PTR == 8)
    1815                 :         x |= x >> 32;
    1816                 : #endif
    1817        14496575 :         x++;
    1818        14496575 :         return (x);
    1819                 : }
    1820                 : 
    1821                 : #ifdef MALLOC_BALANCE
    1822                 : /*
    1823                 :  * Use a simple linear congruential pseudo-random number generator:
    1824                 :  *
    1825                 :  *   prn(y) = (a*x + c) % m
    1826                 :  *
    1827                 :  * where the following constants ensure maximal period:
    1828                 :  *
    1829                 :  *   a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4.
    1830                 :  *   c == Odd number (relatively prime to 2^n).
    1831                 :  *   m == 2^32
    1832                 :  *
    1833                 :  * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints.
    1834                 :  *
    1835                 :  * This choice of m has the disadvantage that the quality of the bits is
    1836                 :  * proportional to bit position.  For example. the lowest bit has a cycle of 2,
    1837                 :  * the next has a cycle of 4, etc.  For this reason, we prefer to use the upper
    1838                 :  * bits.
    1839                 :  */
    1840                 : #  define PRN_DEFINE(suffix, var, a, c)                                 \
    1841                 : static inline void                                                      \
    1842                 : sprn_##suffix(uint32_t seed)                                            \
    1843                 : {                                                                       \
    1844                 :         var = seed;                                                     \
    1845                 : }                                                                       \
    1846                 :                                                                         \
    1847                 : static inline uint32_t                                                  \
    1848                 : prn_##suffix(uint32_t lg_range)                                         \
    1849                 : {                                                                       \
    1850                 :         uint32_t ret, x;                                                \
    1851                 :                                                                         \
    1852                 :         assert(lg_range > 0);                                                \
    1853                 :         assert(lg_range <= 32);                                              \
    1854                 :                                                                         \
    1855                 :         x = (var * (a)) + (c);                                          \
    1856                 :         var = x;                                                        \
    1857                 :         ret = x >> (32 - lg_range);                                       \
    1858                 :                                                                         \
    1859                 :         return (ret);                                                   \
    1860                 : }
    1861                 : #  define SPRN(suffix, seed)    sprn_##suffix(seed)
    1862                 : #  define PRN(suffix, lg_range) prn_##suffix(lg_range)
    1863                 : #endif
    1864                 : 
    1865                 : #ifdef MALLOC_BALANCE
    1866                 : /* Define the PRNG used for arena assignment. */
    1867                 : static __thread uint32_t balance_x;
    1868                 : PRN_DEFINE(balance, balance_x, 1297, 1301)
    1869                 : #endif
    1870                 : 
    1871                 : #ifdef MALLOC_UTRACE
    1872                 : static int
    1873               0 : utrace(const void *addr, size_t len)
    1874                 : {
    1875               0 :         malloc_utrace_t *ut = (malloc_utrace_t *)addr;
    1876                 :         char buf_a[UMAX2S_BUFSIZE];
    1877                 :         char buf_b[UMAX2S_BUFSIZE];
    1878                 : 
    1879               0 :         assert(len == sizeof(malloc_utrace_t));
    1880                 : 
    1881               0 :         if (ut->p == NULL && ut->s == 0 && ut->r == NULL) {
    1882               0 :                 _malloc_message(
    1883               0 :                     umax2s(getpid(), 10, buf_a),
    1884                 :                     " x USER malloc_init()\n", "", "");
    1885               0 :         } else if (ut->p == NULL && ut->r != NULL) {
    1886               0 :                 _malloc_message(
    1887               0 :                     umax2s(getpid(), 10, buf_a),
    1888                 :                     " x USER 0x",
    1889               0 :                     umax2s((uintptr_t)ut->r, 16, buf_b),
    1890                 :                     " = malloc(");
    1891               0 :                 _malloc_message(
    1892               0 :                     umax2s(ut->s, 10, buf_a),
    1893                 :                     ")\n", "", "");
    1894               0 :         } else if (ut->p != NULL && ut->r != NULL) {
    1895               0 :                 _malloc_message(
    1896               0 :                     umax2s(getpid(), 10, buf_a),
    1897                 :                     " x USER 0x",
    1898               0 :                     umax2s((uintptr_t)ut->r, 16, buf_b),
    1899                 :                     " = realloc(0x");
    1900               0 :                 _malloc_message(
    1901               0 :                     umax2s((uintptr_t)ut->p, 16, buf_a),
    1902                 :                     ", ",
    1903               0 :                     umax2s(ut->s, 10, buf_b),
    1904                 :                     ")\n");
    1905                 :         } else {
    1906               0 :                 _malloc_message(
    1907               0 :                     umax2s(getpid(), 10, buf_a),
    1908                 :                     " x USER free(0x",
    1909               0 :                     umax2s((uintptr_t)ut->p, 16, buf_b),
    1910                 :                     ")\n");
    1911                 :         }
    1912                 : 
    1913               0 :         return (0);
    1914                 : }
    1915                 : #endif
    1916                 : 
    1917                 : static inline const char *
    1918               0 : _getprogname(void)
    1919                 : {
    1920                 : 
    1921               0 :         return ("<jemalloc>");
    1922                 : }
    1923                 : 
    1924                 : #ifdef MALLOC_STATS
    1925                 : /*
    1926                 :  * Print to stderr in such a way as to (hopefully) avoid memory allocation.
    1927                 :  */
    1928                 : static void
    1929               0 : malloc_printf(const char *format, ...)
    1930                 : {
    1931                 :         char buf[4096];
    1932                 :         va_list ap;
    1933                 : 
    1934               0 :         va_start(ap, format);
    1935               0 :         vsnprintf(buf, sizeof(buf), format, ap);
    1936               0 :         va_end(ap);
    1937               0 :         _malloc_message(buf, "", "", "");
    1938               0 : }
    1939                 : #endif
    1940                 : 
    1941                 : /******************************************************************************/
    1942                 : 
    1943                 : static inline void
    1944               0 : pages_decommit(void *addr, size_t size)
    1945                 : {
    1946                 : 
    1947                 : #ifdef MOZ_MEMORY_WINDOWS
    1948                 :         VirtualFree(addr, size, MEM_DECOMMIT);
    1949                 : #else
    1950               0 :         if (mmap(addr, size, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1,
    1951                 :             0) == MAP_FAILED)
    1952               0 :                 abort();
    1953                 : #endif
    1954               0 : }
    1955                 : 
    1956                 : static inline void
    1957               0 : pages_commit(void *addr, size_t size)
    1958                 : {
    1959                 : 
    1960                 : #  ifdef MOZ_MEMORY_WINDOWS
    1961                 :         VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
    1962                 : #  else
    1963               0 :         if (mmap(addr, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE |
    1964                 :             MAP_ANON, -1, 0) == MAP_FAILED)
    1965               0 :                 abort();
    1966                 : #  endif
    1967               0 : }
    1968                 : 
    1969                 : static bool
    1970           18669 : base_pages_alloc_mmap(size_t minsize)
    1971                 : {
    1972                 :         bool ret;
    1973                 :         size_t csize;
    1974                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
    1975                 :         size_t pminsize;
    1976                 : #endif
    1977                 :         int pfd;
    1978                 : 
    1979           18669 :         assert(minsize != 0);
    1980           18669 :         csize = CHUNK_CEILING(minsize);
    1981                 : #ifdef MALLOC_PAGEFILE
    1982                 :         if (opt_pagefile) {
    1983                 :                 pfd = pagefile_init(csize);
    1984                 :                 if (pfd == -1)
    1985                 :                         return (true);
    1986                 :         } else
    1987                 : #endif
    1988           18669 :                 pfd = -1;
    1989           18669 :         base_pages = pages_map(NULL, csize, pfd);
    1990           18669 :         if (base_pages == NULL) {
    1991               0 :                 ret = true;
    1992               0 :                 goto RETURN;
    1993                 :         }
    1994           18669 :         base_next_addr = base_pages;
    1995           18669 :         base_past_addr = (void *)((uintptr_t)base_pages + csize);
    1996                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
    1997                 :         /*
    1998                 :          * Leave enough pages for minsize committed, since otherwise they would
    1999                 :          * have to be immediately recommitted.
    2000                 :          */
    2001           18669 :         pminsize = PAGE_CEILING(minsize);
    2002           18669 :         base_next_decommitted = (void *)((uintptr_t)base_pages + pminsize);
    2003                 : #  if defined(MALLOC_DECOMMIT)
    2004                 :         if (pminsize < csize)
    2005                 :                 pages_decommit(base_next_decommitted, csize - pminsize);
    2006                 : #  endif
    2007                 : #  ifdef MALLOC_STATS
    2008           18669 :         base_mapped += csize;
    2009           18669 :         base_committed += pminsize;
    2010                 : #  endif
    2011                 : #endif
    2012                 : 
    2013           18669 :         ret = false;
    2014                 : RETURN:
    2015                 : #ifdef MALLOC_PAGEFILE
    2016                 :         if (pfd != -1)
    2017                 :                 pagefile_close(pfd);
    2018                 : #endif
    2019           18669 :         return (false);
    2020                 : }
    2021                 : 
    2022                 : static bool
    2023           18669 : base_pages_alloc(size_t minsize)
    2024                 : {
    2025                 : 
    2026           18669 :         if (base_pages_alloc_mmap(minsize) == false)
    2027           18669 :                 return (false);
    2028                 : 
    2029               0 :         return (true);
    2030                 : }
    2031                 : 
    2032                 : static void *
    2033           75275 : base_alloc(size_t size)
    2034                 : {
    2035                 :         void *ret;
    2036                 :         size_t csize;
    2037                 : 
    2038                 :         /* Round size up to nearest multiple of the cacheline size. */
    2039           75275 :         csize = CACHELINE_CEILING(size);
    2040                 : 
    2041           75275 :         malloc_mutex_lock(&base_mtx);
    2042                 :         /* Make sure there's enough space for the allocation. */
    2043           75275 :         if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) {
    2044           18669 :                 if (base_pages_alloc(csize)) {
    2045               0 :                         malloc_mutex_unlock(&base_mtx);
    2046               0 :                         return (NULL);
    2047                 :                 }
    2048                 :         }
    2049                 :         /* Allocate. */
    2050           75275 :         ret = base_next_addr;
    2051           75275 :         base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
    2052                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
    2053                 :         /* Make sure enough pages are committed for the new allocation. */
    2054           75275 :         if ((uintptr_t)base_next_addr > (uintptr_t)base_next_decommitted) {
    2055           18669 :                 void *pbase_next_addr =
    2056           18669 :                     (void *)(PAGE_CEILING((uintptr_t)base_next_addr));
    2057                 : 
    2058                 : #  ifdef MALLOC_DECOMMIT
    2059                 :                 pages_commit(base_next_decommitted, (uintptr_t)pbase_next_addr -
    2060                 :                     (uintptr_t)base_next_decommitted);
    2061                 : #  endif
    2062           18669 :                 base_next_decommitted = pbase_next_addr;
    2063                 : #  ifdef MALLOC_STATS
    2064           37338 :                 base_committed += (uintptr_t)pbase_next_addr -
    2065           18669 :                     (uintptr_t)base_next_decommitted;
    2066                 : #  endif
    2067                 :         }
    2068                 : #endif
    2069           75275 :         malloc_mutex_unlock(&base_mtx);
    2070                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
    2071                 : 
    2072           75275 :         return (ret);
    2073                 : }
    2074                 : 
    2075                 : static void *
    2076           37338 : base_calloc(size_t number, size_t size)
    2077                 : {
    2078                 :         void *ret;
    2079                 : 
    2080           37338 :         ret = base_alloc(number * size);
    2081                 : #ifdef MALLOC_VALGRIND
    2082                 :         if (ret != NULL) {
    2083                 :                 VALGRIND_FREELIKE_BLOCK(ret, 0);
    2084                 :                 VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, true);
    2085                 :         }
    2086                 : #endif
    2087           37338 :         memset(ret, 0, number * size);
    2088                 : 
    2089           37338 :         return (ret);
    2090                 : }
    2091                 : 
    2092                 : static extent_node_t *
    2093            4554 : base_node_alloc(void)
    2094                 : {
    2095                 :         extent_node_t *ret;
    2096                 : 
    2097            4554 :         malloc_mutex_lock(&base_mtx);
    2098            4554 :         if (base_nodes != NULL) {
    2099            3955 :                 ret = base_nodes;
    2100            3955 :                 base_nodes = *(extent_node_t **)ret;
    2101                 :                 VALGRIND_FREELIKE_BLOCK(ret, 0);
    2102                 :                 VALGRIND_MALLOCLIKE_BLOCK(ret, sizeof(extent_node_t), 0, false);
    2103            3955 :                 malloc_mutex_unlock(&base_mtx);
    2104                 :         } else {
    2105             599 :                 malloc_mutex_unlock(&base_mtx);
    2106             599 :                 ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
    2107                 :         }
    2108                 : 
    2109            4554 :         return (ret);
    2110                 : }
    2111                 : 
    2112                 : static void
    2113            4554 : base_node_dealloc(extent_node_t *node)
    2114                 : {
    2115                 : 
    2116            4554 :         malloc_mutex_lock(&base_mtx);
    2117                 :         VALGRIND_FREELIKE_BLOCK(node, 0);
    2118                 :         VALGRIND_MALLOCLIKE_BLOCK(node, sizeof(extent_node_t *), 0, false);
    2119            4554 :         *(extent_node_t **)node = base_nodes;
    2120            4554 :         base_nodes = node;
    2121            4554 :         malloc_mutex_unlock(&base_mtx);
    2122            4554 : }
    2123                 : 
    2124                 : /******************************************************************************/
    2125                 : 
    2126                 : #ifdef MALLOC_STATS
    2127                 : static void
    2128               0 : stats_print(arena_t *arena)
    2129                 : {
    2130                 :         unsigned i, gap_start;
    2131                 : 
    2132                 : #ifdef MOZ_MEMORY_WINDOWS
    2133                 :         malloc_printf("dirty: %Iu page%s dirty, %I64u sweep%s,"
    2134                 :             " %I64u madvise%s, %I64u page%s purged\n",
    2135                 :             arena->ndirty, arena->ndirty == 1 ? "" : "s",
    2136                 :             arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
    2137                 :             arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
    2138                 :             arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
    2139                 : #  ifdef MALLOC_DECOMMIT
    2140                 :         malloc_printf("decommit: %I64u decommit%s, %I64u commit%s,"
    2141                 :             " %I64u page%s decommitted\n",
    2142                 :             arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
    2143                 :             arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
    2144                 :             arena->stats.decommitted,
    2145                 :             (arena->stats.decommitted == 1) ? "" : "s");
    2146                 : #  endif
    2147                 : 
    2148                 :         malloc_printf("            allocated      nmalloc      ndalloc\n");
    2149                 :         malloc_printf("small:   %12Iu %12I64u %12I64u\n",
    2150                 :             arena->stats.allocated_small, arena->stats.nmalloc_small,
    2151                 :             arena->stats.ndalloc_small);
    2152                 :         malloc_printf("large:   %12Iu %12I64u %12I64u\n",
    2153                 :             arena->stats.allocated_large, arena->stats.nmalloc_large,
    2154                 :             arena->stats.ndalloc_large);
    2155                 :         malloc_printf("total:   %12Iu %12I64u %12I64u\n",
    2156                 :             arena->stats.allocated_small + arena->stats.allocated_large,
    2157                 :             arena->stats.nmalloc_small + arena->stats.nmalloc_large,
    2158                 :             arena->stats.ndalloc_small + arena->stats.ndalloc_large);
    2159                 :         malloc_printf("mapped:  %12Iu\n", arena->stats.mapped);
    2160                 : #else
    2161               0 :         malloc_printf("dirty: %zu page%s dirty, %llu sweep%s,"
    2162                 :             " %llu madvise%s, %llu page%s purged\n",
    2163               0 :             arena->ndirty, arena->ndirty == 1 ? "" : "s",
    2164               0 :             arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
    2165               0 :             arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
    2166               0 :             arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
    2167                 : #  ifdef MALLOC_DECOMMIT
    2168                 :         malloc_printf("decommit: %llu decommit%s, %llu commit%s,"
    2169                 :             " %llu page%s decommitted\n",
    2170                 :             arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
    2171                 :             arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
    2172                 :             arena->stats.decommitted,
    2173                 :             (arena->stats.decommitted == 1) ? "" : "s");
    2174                 : #  endif
    2175                 : 
    2176               0 :         malloc_printf("            allocated      nmalloc      ndalloc\n");
    2177               0 :         malloc_printf("small:   %12zu %12llu %12llu\n",
    2178                 :             arena->stats.allocated_small, arena->stats.nmalloc_small,
    2179                 :             arena->stats.ndalloc_small);
    2180               0 :         malloc_printf("large:   %12zu %12llu %12llu\n",
    2181                 :             arena->stats.allocated_large, arena->stats.nmalloc_large,
    2182                 :             arena->stats.ndalloc_large);
    2183               0 :         malloc_printf("total:   %12zu %12llu %12llu\n",
    2184               0 :             arena->stats.allocated_small + arena->stats.allocated_large,
    2185               0 :             arena->stats.nmalloc_small + arena->stats.nmalloc_large,
    2186               0 :             arena->stats.ndalloc_small + arena->stats.ndalloc_large);
    2187               0 :         malloc_printf("mapped:  %12zu\n", arena->stats.mapped);
    2188                 : #endif
    2189               0 :         malloc_printf("bins:     bin   size regs pgs  requests   newruns"
    2190                 :             "    reruns maxruns curruns\n");
    2191               0 :         for (i = 0, gap_start = UINT_MAX; i < ntbins + nqbins + nsbins; i++) {
    2192               0 :                 if (arena->bins[i].stats.nrequests == 0) {
    2193               0 :                         if (gap_start == UINT_MAX)
    2194               0 :                                 gap_start = i;
    2195                 :                 } else {
    2196               0 :                         if (gap_start != UINT_MAX) {
    2197               0 :                                 if (i > gap_start + 1) {
    2198                 :                                         /* Gap of more than one size class. */
    2199               0 :                                         malloc_printf("[%u..%u]\n",
    2200                 :                                             gap_start, i - 1);
    2201                 :                                 } else {
    2202                 :                                         /* Gap of one size class. */
    2203               0 :                                         malloc_printf("[%u]\n", gap_start);
    2204                 :                                 }
    2205               0 :                                 gap_start = UINT_MAX;
    2206                 :                         }
    2207               0 :                         malloc_printf(
    2208                 : #if defined(MOZ_MEMORY_WINDOWS)
    2209                 :                             "%13u %1s %4u %4u %3u %9I64u %9I64u"
    2210                 :                             " %9I64u %7u %7u\n",
    2211                 : #else
    2212                 :                             "%13u %1s %4u %4u %3u %9llu %9llu"
    2213                 :                             " %9llu %7lu %7lu\n",
    2214                 : #endif
    2215                 :                             i,
    2216               0 :                             i < ntbins ? "T" : i < ntbins + nqbins ? "Q" : "S",
    2217                 :                             arena->bins[i].reg_size,
    2218                 :                             arena->bins[i].nregs,
    2219               0 :                             arena->bins[i].run_size >> pagesize_2pow,
    2220                 :                             arena->bins[i].stats.nrequests,
    2221                 :                             arena->bins[i].stats.nruns,
    2222                 :                             arena->bins[i].stats.reruns,
    2223                 :                             arena->bins[i].stats.highruns,
    2224                 :                             arena->bins[i].stats.curruns);
    2225                 :                 }
    2226                 :         }
    2227               0 :         if (gap_start != UINT_MAX) {
    2228               0 :                 if (i > gap_start + 1) {
    2229                 :                         /* Gap of more than one size class. */
    2230               0 :                         malloc_printf("[%u..%u]\n", gap_start, i - 1);
    2231                 :                 } else {
    2232                 :                         /* Gap of one size class. */
    2233               0 :                         malloc_printf("[%u]\n", gap_start);
    2234                 :                 }
    2235                 :         }
    2236               0 : }
    2237                 : #endif
    2238                 : 
    2239                 : /*
    2240                 :  * End Utility functions/macros.
    2241                 :  */
    2242                 : /******************************************************************************/
    2243                 : /*
    2244                 :  * Begin extent tree code.
    2245                 :  */
    2246                 : 
    2247                 : static inline int
    2248               0 : extent_szad_comp(extent_node_t *a, extent_node_t *b)
    2249                 : {
    2250                 :         int ret;
    2251               0 :         size_t a_size = a->size;
    2252               0 :         size_t b_size = b->size;
    2253                 : 
    2254               0 :         ret = (a_size > b_size) - (a_size < b_size);
    2255               0 :         if (ret == 0) {
    2256               0 :                 uintptr_t a_addr = (uintptr_t)a->addr;
    2257               0 :                 uintptr_t b_addr = (uintptr_t)b->addr;
    2258                 : 
    2259               0 :                 ret = (a_addr > b_addr) - (a_addr < b_addr);
    2260                 :         }
    2261                 : 
    2262               0 :         return (ret);
    2263                 : }
    2264                 : 
    2265                 : /* Wrap red-black tree macros in functions. */
    2266               0 : rb_wrap(static, extent_tree_szad_, extent_tree_t, extent_node_t,
    2267               0 :     link_szad, extent_szad_comp)
    2268                 : 
    2269                 : static inline int
    2270           28261 : extent_ad_comp(extent_node_t *a, extent_node_t *b)
    2271                 : {
    2272           28261 :         uintptr_t a_addr = (uintptr_t)a->addr;
    2273           28261 :         uintptr_t b_addr = (uintptr_t)b->addr;
    2274                 : 
    2275           28261 :         return ((a_addr > b_addr) - (a_addr < b_addr));
    2276                 : }
    2277                 : 
    2278                 : /* Wrap red-black tree macros in functions. */
    2279           33897 : rb_wrap(static, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
    2280           33897 :     extent_ad_comp)
    2281                 : 
    2282                 : /*
    2283                 :  * End extent tree code.
    2284                 :  */
    2285                 : /******************************************************************************/
    2286                 : /*
    2287                 :  * Begin chunk management functions.
    2288                 :  */
    2289                 : 
    2290                 : #ifdef MOZ_MEMORY_WINDOWS
    2291                 : 
    2292                 : static void *
    2293                 : pages_map(void *addr, size_t size, int pfd)
    2294                 : {
    2295                 :         void *ret = NULL;
    2296                 :         ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
    2297                 :             PAGE_READWRITE);
    2298                 :         return (ret);
    2299                 : }
    2300                 : 
    2301                 : static void
    2302                 : pages_unmap(void *addr, size_t size)
    2303                 : {
    2304                 :         if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
    2305                 :                 _malloc_message(_getprogname(),
    2306                 :                     ": (malloc) Error in VirtualFree()\n", "", "");
    2307                 :                 if (opt_abort)
    2308                 :                         abort();
    2309                 :         }
    2310                 : }
    2311                 : #else
    2312                 : #ifdef JEMALLOC_USES_MAP_ALIGN
    2313                 : static void *
    2314                 : pages_map_align(size_t size, int pfd, size_t alignment)
    2315                 : {
    2316                 :         void *ret;
    2317                 : 
    2318                 :         /*
    2319                 :          * We don't use MAP_FIXED here, because it can cause the *replacement*
    2320                 :          * of existing mappings, and we only want to create new mappings.
    2321                 :          */
    2322                 : #ifdef MALLOC_PAGEFILE
    2323                 :         if (pfd != -1) {
    2324                 :                 ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
    2325                 :                     MAP_NOSYNC | MAP_ALIGN, pfd, 0);
    2326                 :         } else
    2327                 : #endif
    2328                 :                {
    2329                 :                 ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
    2330                 :                     MAP_NOSYNC | MAP_ALIGN | MAP_ANON, -1, 0);
    2331                 :         }
    2332                 :         assert(ret != NULL);
    2333                 : 
    2334                 :         if (ret == MAP_FAILED)
    2335                 :                 ret = NULL;
    2336                 :         return (ret);
    2337                 : }
    2338                 : #endif
    2339                 : 
    2340                 : static void *
    2341           64774 : pages_map(void *addr, size_t size, int pfd)
    2342                 : {
    2343                 :         void *ret;
    2344                 : #if defined(__ia64__)
    2345                 :         /*
    2346                 :          * The JS engine assumes that all allocated pointers have their high 17 bits clear,
    2347                 :          * which ia64's mmap doesn't support directly. However, we can emulate it by passing
    2348                 :          * mmap an "addr" parameter with those bits clear. The mmap will return that address,
    2349                 :          * or the nearest available memory above that address, providing a near-guarantee
    2350                 :          * that those bits are clear. If they are not, we return NULL below to indicate
    2351                 :          * out-of-memory.
    2352                 :          * 
    2353                 :          * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual 
    2354                 :          * address space.
    2355                 :          * 
    2356                 :          * See Bug 589735 for more information.
    2357                 :          */
    2358                 :         bool check_placement = true;
    2359                 :         if (addr == NULL) {
    2360                 :                 addr = (void*)0x0000070000000000;
    2361                 :                 check_placement = false;
    2362                 :         }
    2363                 : #endif
    2364                 : 
    2365                 :         /*
    2366                 :          * We don't use MAP_FIXED here, because it can cause the *replacement*
    2367                 :          * of existing mappings, and we only want to create new mappings.
    2368                 :          */
    2369                 : #ifdef MALLOC_PAGEFILE
    2370                 :         if (pfd != -1) {
    2371                 :                 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
    2372                 :                     MAP_NOSYNC, pfd, 0);
    2373                 :         } else
    2374                 : #endif
    2375                 :                {
    2376           64774 :                 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
    2377                 :                     MAP_ANON, -1, 0);
    2378                 :         }
    2379           64774 :         assert(ret != NULL);
    2380                 : 
    2381           64774 :         if (ret == MAP_FAILED) {
    2382              18 :                 ret = NULL;
    2383                 :         }
    2384                 : #if defined(__ia64__)
    2385                 :         /* 
    2386                 :          * If the allocated memory doesn't have its upper 17 bits clear, consider it 
    2387                 :          * as out of memory.
    2388                 :         */
    2389                 :         else if ((long long)ret & 0xffff800000000000) {
    2390                 :                 munmap(ret, size);
    2391                 :                 ret = NULL;
    2392                 :         }
    2393                 :         /* If the caller requested a specific memory location, verify that's what mmap returned. */
    2394                 :         else if (check_placement && ret != addr) {
    2395                 : #else
    2396           64756 :         else if (addr != NULL && ret != addr) {
    2397                 : #endif
    2398                 :                 /*
    2399                 :                  * We succeeded in mapping memory, but not in the right place.
    2400                 :                  */
    2401               0 :                 if (munmap(ret, size) == -1) {
    2402                 :                         char buf[STRERROR_BUF];
    2403                 : 
    2404               0 :                         strerror_r(errno, buf, sizeof(buf));
    2405               0 :                         _malloc_message(_getprogname(),
    2406                 :                             ": (malloc) Error in munmap(): ", buf, "\n");
    2407               0 :                         if (opt_abort)
    2408               0 :                                 abort();
    2409                 :                 }
    2410               0 :                 ret = NULL;
    2411                 :         }
    2412                 : 
    2413                 : #if defined(__ia64__)
    2414                 :         assert(ret == NULL || (!check_placement && ret != NULL)
    2415                 :             || (check_placement && ret == addr));
    2416                 : #else
    2417           64774 :         assert(ret == NULL || (addr == NULL && ret != addr)
    2418                 :             || (addr != NULL && ret == addr));
    2419                 : #endif
    2420           64774 :         return (ret);
    2421                 : }
    2422                 : 
    2423                 : static void
    2424           27462 : pages_unmap(void *addr, size_t size)
    2425                 : {
    2426                 : 
    2427           27462 :         if (munmap(addr, size) == -1) {
    2428                 :                 char buf[STRERROR_BUF];
    2429                 : 
    2430               0 :                 strerror_r(errno, buf, sizeof(buf));
    2431               0 :                 _malloc_message(_getprogname(),
    2432                 :                     ": (malloc) Error in munmap(): ", buf, "\n");
    2433               0 :                 if (opt_abort)
    2434               0 :                         abort();
    2435                 :         }
    2436           27462 : }
    2437                 : #endif
    2438                 : 
    2439                 : #ifdef MOZ_MEMORY_DARWIN
    2440                 : #define VM_COPY_MIN (pagesize << 5)
    2441                 : static inline void
    2442                 : pages_copy(void *dest, const void *src, size_t n)
    2443                 : {
    2444                 : 
    2445                 :         assert((void *)((uintptr_t)dest & ~pagesize_mask) == dest);
    2446                 :         assert(n >= VM_COPY_MIN);
    2447                 :         assert((void *)((uintptr_t)src & ~pagesize_mask) == src);
    2448                 : 
    2449                 :         vm_copy(mach_task_self(), (vm_address_t)src, (vm_size_t)n,
    2450                 :             (vm_address_t)dest);
    2451                 : }
    2452                 : #endif
    2453                 : 
    2454                 : #ifdef MALLOC_VALIDATE
    2455                 : static inline malloc_rtree_t *
    2456           18669 : malloc_rtree_new(unsigned bits)
    2457                 : {
    2458                 :         malloc_rtree_t *ret;
    2459                 :         unsigned bits_per_level, height, i;
    2460                 : 
    2461           37338 :         bits_per_level = ffs(pow2_ceil((MALLOC_RTREE_NODESIZE /
    2462           18669 :             sizeof(void *)))) - 1;
    2463           18669 :         height = bits / bits_per_level;
    2464           18669 :         if (height * bits_per_level != bits)
    2465               0 :                 height++;
    2466           18669 :         assert(height * bits_per_level >= bits);
    2467                 : 
    2468           18669 :         ret = (malloc_rtree_t*)base_calloc(1, sizeof(malloc_rtree_t) +
    2469                 :             (sizeof(unsigned) * (height - 1)));
    2470           18669 :         if (ret == NULL)
    2471               0 :                 return (NULL);
    2472                 : 
    2473           18669 :         malloc_spin_init(&ret->lock);
    2474           18669 :         ret->height = height;
    2475           18669 :         if (bits_per_level * height > bits)
    2476               0 :                 ret->level2bits[0] = bits % bits_per_level;
    2477                 :         else
    2478           18669 :                 ret->level2bits[0] = bits_per_level;
    2479           18669 :         for (i = 1; i < height; i++)
    2480               0 :                 ret->level2bits[i] = bits_per_level;
    2481                 : 
    2482           18669 :         ret->root = (void**)base_calloc(1, sizeof(void *) << ret->level2bits[0]);
    2483           18669 :         if (ret->root == NULL) {
    2484                 :                 /*
    2485                 :                  * We leak the rtree here, since there's no generic base
    2486                 :                  * deallocation.
    2487                 :                  */
    2488               0 :                 return (NULL);
    2489                 :         }
    2490                 : 
    2491           18669 :         return (ret);
    2492                 : }
    2493                 : 
    2494                 : #define MALLOC_RTREE_GET_GENERATE(f)                                    \
    2495                 : /* The least significant bits of the key are ignored. */                \
    2496                 : static inline void *                                                    \
    2497                 : f(malloc_rtree_t *rtree, uintptr_t key)                                 \
    2498                 : {                                                                       \
    2499                 :         void *ret;                                                      \
    2500                 :         uintptr_t subkey;                                               \
    2501                 :         unsigned i, lshift, height, bits;                               \
    2502                 :         void **node, **child;                                           \
    2503                 :                                                                         \
    2504                 :         MALLOC_RTREE_LOCK(&rtree->lock);                         \
    2505                 :         for (i = lshift = 0, height = rtree->height, node = rtree->root;\
    2506                 :             i < height - 1;                                          \
    2507                 :             i++, lshift += bits, node = child) {                        \
    2508                 :                 bits = rtree->level2bits[i];                         \
    2509                 :                 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);       \
    2510                 :                 child = (void**)node[subkey];                           \
    2511                 :                 if (child == NULL) {                                    \
    2512                 :                         MALLOC_RTREE_UNLOCK(&rtree->lock);               \
    2513                 :                         return (NULL);                                  \
    2514                 :                 }                                                       \
    2515                 :         }                                                               \
    2516                 :                                                                         \
    2517                 :         /*                                                              \
    2518                 :          * node is a leaf, so it contains values rather than node       \
    2519                 :          * pointers.                                                    \
    2520                 :          */                                                             \
    2521                 :         bits = rtree->level2bits[i];                                 \
    2522                 :         subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);               \
    2523                 :         ret = node[subkey];                                             \
    2524                 :         MALLOC_RTREE_UNLOCK(&rtree->lock);                               \
    2525                 :                                                                         \
    2526                 :         MALLOC_RTREE_GET_VALIDATE                                       \
    2527                 :         return (ret);                                                   \
    2528                 : }
    2529                 : 
    2530                 : #ifdef MALLOC_DEBUG
    2531                 : #  define MALLOC_RTREE_LOCK(l)          malloc_spin_lock(l)
    2532                 : #  define MALLOC_RTREE_UNLOCK(l)        malloc_spin_unlock(l)
    2533                 : #  define MALLOC_RTREE_GET_VALIDATE
    2534               0 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get_locked)
    2535                 : #  undef MALLOC_RTREE_LOCK
    2536                 : #  undef MALLOC_RTREE_UNLOCK
    2537                 : #  undef MALLOC_RTREE_GET_VALIDATE
    2538                 : #endif
    2539                 : 
    2540                 : #define MALLOC_RTREE_LOCK(l)
    2541                 : #define MALLOC_RTREE_UNLOCK(l)
    2542                 : #ifdef MALLOC_DEBUG
    2543                 :    /*
    2544                 :     * Suppose that it were possible for a jemalloc-allocated chunk to be
    2545                 :     * munmap()ped, followed by a different allocator in another thread re-using
    2546                 :     * overlapping virtual memory, all without invalidating the cached rtree
    2547                 :     * value.  The result would be a false positive (the rtree would claim that
    2548                 :     * jemalloc owns memory that it had actually discarded).  I don't think this
    2549                 :     * scenario is possible, but the following assertion is a prudent sanity
    2550                 :     * check.
    2551                 :     */
    2552                 : #  define MALLOC_RTREE_GET_VALIDATE                                     \
    2553                 :         assert(malloc_rtree_get_locked(rtree, key) == ret);
    2554                 : #else
    2555                 : #  define MALLOC_RTREE_GET_VALIDATE
    2556                 : #endif
    2557               0 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get)
    2558                 : #undef MALLOC_RTREE_LOCK
    2559                 : #undef MALLOC_RTREE_UNLOCK
    2560                 : #undef MALLOC_RTREE_GET_VALIDATE
    2561                 : 
    2562                 : static inline bool
    2563           33224 : malloc_rtree_set(malloc_rtree_t *rtree, uintptr_t key, void *val)
    2564                 : {
    2565                 :         uintptr_t subkey;
    2566                 :         unsigned i, lshift, height, bits;
    2567                 :         void **node, **child;
    2568                 : 
    2569           33224 :         malloc_spin_lock(&rtree->lock);
    2570           66448 :         for (i = lshift = 0, height = rtree->height, node = rtree->root;
    2571           33224 :             i < height - 1;
    2572               0 :             i++, lshift += bits, node = child) {
    2573               0 :                 bits = rtree->level2bits[i];
    2574               0 :                 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
    2575               0 :                 child = (void**)node[subkey];
    2576               0 :                 if (child == NULL) {
    2577               0 :                         child = (void**)base_calloc(1, sizeof(void *) <<
    2578               0 :                             rtree->level2bits[i+1]);
    2579               0 :                         if (child == NULL) {
    2580               0 :                                 malloc_spin_unlock(&rtree->lock);
    2581               0 :                                 return (true);
    2582                 :                         }
    2583               0 :                         node[subkey] = child;
    2584                 :                 }
    2585                 :         }
    2586                 : 
    2587                 :         /* node is a leaf, so it contains values rather than node pointers. */
    2588           33224 :         bits = rtree->level2bits[i];
    2589           33224 :         subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
    2590           33224 :         node[subkey] = val;
    2591           33224 :         malloc_spin_unlock(&rtree->lock);
    2592                 : 
    2593           33224 :         return (false);
    2594                 : }
    2595                 : #endif
    2596                 : 
    2597                 : #if defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)
    2598                 : 
    2599                 : /* Allocate an aligned chunk while maintaining a 1:1 correspondence between
    2600                 :  * mmap and unmap calls.  This is important on Windows, but not elsewhere. */
    2601                 : static void *
    2602                 : chunk_alloc_mmap(size_t size, bool pagefile)
    2603                 : {
    2604                 :         void *ret;
    2605                 : #ifndef JEMALLOC_USES_MAP_ALIGN
    2606                 :         size_t offset;
    2607                 : #endif
    2608                 :         int pfd;
    2609                 : 
    2610                 : #ifdef MALLOC_PAGEFILE
    2611                 :         if (opt_pagefile && pagefile) {
    2612                 :                 pfd = pagefile_init(size);
    2613                 :                 if (pfd == -1)
    2614                 :                         return (NULL);
    2615                 :         } else
    2616                 : #endif
    2617                 :                 pfd = -1;
    2618                 : 
    2619                 : #ifdef JEMALLOC_USES_MAP_ALIGN
    2620                 :         ret = pages_map_align(size, pfd, chunksize);
    2621                 : #else
    2622                 :         ret = pages_map(NULL, size, pfd);
    2623                 :         if (ret == NULL)
    2624                 :                 goto RETURN;
    2625                 : 
    2626                 :         offset = CHUNK_ADDR2OFFSET(ret);
    2627                 :         if (offset != 0) {
    2628                 :                 /* Deallocate, then try to allocate at (ret + size - offset). */
    2629                 :                 pages_unmap(ret, size);
    2630                 :                 ret = pages_map((void *)((uintptr_t)ret + size - offset), size,
    2631                 :                     pfd);
    2632                 :                 while (ret == NULL) {
    2633                 :                         /*
    2634                 :                          * Over-allocate in order to map a memory region that
    2635                 :                          * is definitely large enough.
    2636                 :                          */
    2637                 :                         ret = pages_map(NULL, size + chunksize, -1);
    2638                 :                         if (ret == NULL)
    2639                 :                                 goto RETURN;
    2640                 :                         /*
    2641                 :                          * Deallocate, then allocate the correct size, within
    2642                 :                          * the over-sized mapping.
    2643                 :                          */
    2644                 :                         offset = CHUNK_ADDR2OFFSET(ret);
    2645                 :                         pages_unmap(ret, size + chunksize);
    2646                 :                         if (offset == 0)
    2647                 :                                 ret = pages_map(ret, size, pfd);
    2648                 :                         else {
    2649                 :                                 ret = pages_map((void *)((uintptr_t)ret +
    2650                 :                                     chunksize - offset), size, pfd);
    2651                 :                         }
    2652                 :                         /*
    2653                 :                          * Failure here indicates a race with another thread, so
    2654                 :                          * try again.
    2655                 :                          */
    2656                 :                 }
    2657                 :         }
    2658                 : RETURN:
    2659                 : #endif
    2660                 : #ifdef MALLOC_PAGEFILE
    2661                 :         if (pfd != -1)
    2662                 :                 pagefile_close(pfd);
    2663                 : #endif
    2664                 : #ifdef MALLOC_STATS
    2665                 :         if (ret != NULL)
    2666                 :                 stats_chunks.nchunks += (size / chunksize);
    2667                 : #endif
    2668                 :         return (ret);
    2669                 : }
    2670                 : 
    2671                 : #else /* ! (defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)) */
    2672                 : 
    2673                 : /*
    2674                 :  * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
    2675                 :  * potentially avoid some system calls.
    2676                 :  */
    2677                 : #ifndef NO_TLS
    2678                 : static __thread bool    mmap_unaligned_tls __attribute__((tls_model("initial-exec")));
    2679                 : #define MMAP_UNALIGNED_GET()    mmap_unaligned_tls
    2680                 : #define MMAP_UNALIGNED_SET(v)   do {                                    \
    2681                 :         mmap_unaligned_tls = (v);                                       \
    2682                 : } while (0)
    2683                 : #else
    2684                 : #define NEEDS_PTHREAD_MMAP_UNALIGNED_TSD
    2685                 : static pthread_key_t    mmap_unaligned_tsd;
    2686                 : #define MMAP_UNALIGNED_GET()    ((bool)pthread_getspecific(mmap_unaligned_tsd))
    2687                 : #define MMAP_UNALIGNED_SET(v)   do {                                    \
    2688                 :         pthread_setspecific(mmap_unaligned_tsd, (void *)(v));           \
    2689                 : } while (0)
    2690                 : #endif
    2691                 : 
    2692                 : /* chunk_alloc_mmap_slow and chunk_alloc_mmap were cherry-picked from upstream
    2693                 :  * jemalloc 2.2.3 to fix Mozilla bug 694896, enable jemalloc on Mac 10.7. */
    2694                 : 
    2695                 : static void *
    2696            2961 : chunk_alloc_mmap_slow(size_t size, bool unaligned)
    2697                 : {
    2698                 :         void *ret;
    2699                 :         size_t offset;
    2700                 : 
    2701                 :         /* Beware size_t wrap-around. */
    2702            2961 :         if (size + chunksize <= size)
    2703               0 :                 return (NULL);
    2704                 : 
    2705            2961 :         ret = pages_map(NULL, size + chunksize, -1);
    2706            2961 :         if (ret == NULL)
    2707               0 :                 return (NULL);
    2708                 : 
    2709                 :         /* Clean up unneeded leading/trailing space. */
    2710            2961 :         offset = CHUNK_ADDR2OFFSET(ret);
    2711            2961 :         if (offset != 0) {
    2712                 :                 /* Note that mmap() returned an unaligned mapping. */
    2713              10 :                 unaligned = true;
    2714                 : 
    2715                 :                 /* Leading space. */
    2716              10 :                 pages_unmap(ret, chunksize - offset);
    2717                 : 
    2718              10 :                 ret = (void *)((uintptr_t)ret +
    2719              10 :                     (chunksize - offset));
    2720                 : 
    2721                 :                 /* Trailing space. */
    2722              10 :                 pages_unmap((void *)((uintptr_t)ret + size),
    2723                 :                     offset);
    2724                 :         } else {
    2725                 :                 /* Trailing space only. */
    2726            2951 :                 pages_unmap((void *)((uintptr_t)ret + size),
    2727                 :                     chunksize);
    2728                 :         }
    2729                 : 
    2730                 :         /*
    2731                 :          * If mmap() returned an aligned mapping, reset mmap_unaligned so that
    2732                 :          * the next chunk_alloc_mmap() execution tries the fast allocation
    2733                 :          * method.
    2734                 :          */
    2735            2961 :         if (unaligned == false)
    2736            2951 :                 MMAP_UNALIGNED_SET(false);
    2737                 : 
    2738            2961 :         return (ret);
    2739                 : }
    2740                 : 
    2741                 : static void *
    2742           27428 : chunk_alloc_mmap(size_t size, bool pagefile)
    2743                 : {
    2744                 :         void *ret;
    2745                 : 
    2746                 :         /*
    2747                 :          * Ideally, there would be a way to specify alignment to mmap() (like
    2748                 :          * NetBSD has), but in the absence of such a feature, we have to work
    2749                 :          * hard to efficiently create aligned mappings.  The reliable, but
    2750                 :          * slow method is to create a mapping that is over-sized, then trim the
    2751                 :          * excess.  However, that always results in at least one call to
    2752                 :          * pages_unmap().
    2753                 :          *
    2754                 :          * A more optimistic approach is to try mapping precisely the right
    2755                 :          * amount, then try to append another mapping if alignment is off.  In
    2756                 :          * practice, this works out well as long as the application is not
    2757                 :          * interleaving mappings via direct mmap() calls.  If we do run into a
    2758                 :          * situation where there is an interleaved mapping and we are unable to
    2759                 :          * extend an unaligned mapping, our best option is to switch to the
    2760                 :          * slow method until mmap() returns another aligned mapping.  This will
    2761                 :          * tend to leave a gap in the memory map that is too small to cause
    2762                 :          * later problems for the optimistic method.
    2763                 :          *
    2764                 :          * Another possible confounding factor is address space layout
    2765                 :          * randomization (ASLR), which causes mmap(2) to disregard the
    2766                 :          * requested address.  mmap_unaligned tracks whether the previous
    2767                 :          * chunk_alloc_mmap() execution received any unaligned or relocated
    2768                 :          * mappings, and if so, the current execution will immediately fall
    2769                 :          * back to the slow method.  However, we keep track of whether the fast
    2770                 :          * method would have succeeded, and if so, we make a note to try the
    2771                 :          * fast method next time.
    2772                 :          */
    2773                 : 
    2774           27428 :         if (MMAP_UNALIGNED_GET() == false) {
    2775                 :                 size_t offset;
    2776                 : 
    2777           24467 :                 ret = pages_map(NULL, size, -1);
    2778           24467 :                 if (ret == NULL)
    2779              18 :                         return (NULL);
    2780                 : 
    2781           24449 :                 offset = CHUNK_ADDR2OFFSET(ret);
    2782           24449 :                 if (offset != 0) {
    2783           18677 :                         MMAP_UNALIGNED_SET(true);
    2784                 :                         /* Try to extend chunk boundary. */
    2785           18677 :                         if (pages_map((void *)((uintptr_t)ret + size),
    2786                 :                             chunksize - offset, -1) == NULL) {
    2787                 :                                 /*
    2788                 :                                  * Extension failed.  Clean up, then revert to
    2789                 :                                  * the reliable-but-expensive method.
    2790                 :                                  */
    2791               0 :                                 pages_unmap(ret, size);
    2792               0 :                                 ret = chunk_alloc_mmap_slow(size, true);
    2793                 :                         } else {
    2794                 :                                 /* Clean up unneeded leading space. */
    2795           18677 :                                 pages_unmap(ret, chunksize - offset);
    2796           18677 :                                 ret = (void *)((uintptr_t)ret + (chunksize -
    2797                 :                                     offset));
    2798                 :                         }
    2799                 :                 }
    2800                 :         } else
    2801            2961 :                 ret = chunk_alloc_mmap_slow(size, false);
    2802                 : 
    2803           27410 :         return (ret);
    2804                 : }
    2805                 : 
    2806                 : #endif /* defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE) */
    2807                 : 
    2808                 : #ifdef MALLOC_PAGEFILE
    2809                 : static int
    2810                 : pagefile_init(size_t size)
    2811                 : {
    2812                 :         int ret;
    2813                 :         size_t i;
    2814                 :         char pagefile_path[PATH_MAX];
    2815                 :         char zbuf[MALLOC_PAGEFILE_WRITE_SIZE];
    2816                 : 
    2817                 :         /*
    2818                 :          * Create a temporary file, then immediately unlink it so that it will
    2819                 :          * not persist.
    2820                 :          */
    2821                 :         strcpy(pagefile_path, pagefile_templ);
    2822                 :         ret = mkstemp(pagefile_path);
    2823                 :         if (ret == -1)
    2824                 :                 return (ret);
    2825                 :         if (unlink(pagefile_path)) {
    2826                 :                 char buf[STRERROR_BUF];
    2827                 : 
    2828                 :                 strerror_r(errno, buf, sizeof(buf));
    2829                 :                 _malloc_message(_getprogname(), ": (malloc) Error in unlink(\"",
    2830                 :                     pagefile_path, "\"):");
    2831                 :                 _malloc_message(buf, "\n", "", "");
    2832                 :                 if (opt_abort)
    2833                 :                         abort();
    2834                 :         }
    2835                 : 
    2836                 :         /*
    2837                 :          * Write sequential zeroes to the file in order to assure that disk
    2838                 :          * space is committed, with minimal fragmentation.  It would be
    2839                 :          * sufficient to write one zero per disk block, but that potentially
    2840                 :          * results in more system calls, for no real gain.
    2841                 :          */
    2842                 :         memset(zbuf, 0, sizeof(zbuf));
    2843                 :         for (i = 0; i < size; i += sizeof(zbuf)) {
    2844                 :                 if (write(ret, zbuf, sizeof(zbuf)) != sizeof(zbuf)) {
    2845                 :                         if (errno != ENOSPC) {
    2846                 :                                 char buf[STRERROR_BUF];
    2847                 : 
    2848                 :                                 strerror_r(errno, buf, sizeof(buf));
    2849                 :                                 _malloc_message(_getprogname(),
    2850                 :                                     ": (malloc) Error in write(): ", buf, "\n");
    2851                 :                                 if (opt_abort)
    2852                 :                                         abort();
    2853                 :                         }
    2854                 :                         pagefile_close(ret);
    2855                 :                         return (-1);
    2856                 :                 }
    2857                 :         }
    2858                 : 
    2859                 :         return (ret);
    2860                 : }
    2861                 : 
    2862                 : static void
    2863                 : pagefile_close(int pfd)
    2864                 : {
    2865                 : 
    2866                 :         if (close(pfd)) {
    2867                 :                 char buf[STRERROR_BUF];
    2868                 : 
    2869                 :                 strerror_r(errno, buf, sizeof(buf));
    2870                 :                 _malloc_message(_getprogname(),
    2871                 :                     ": (malloc) Error in close(): ", buf, "\n");
    2872                 :                 if (opt_abort)
    2873                 :                         abort();
    2874                 :         }
    2875                 : }
    2876                 : #endif
    2877                 : 
    2878                 : static void *
    2879           27428 : chunk_alloc(size_t size, bool zero, bool pagefile)
    2880                 : {
    2881                 :         void *ret;
    2882                 : 
    2883           27428 :         assert(size != 0);
    2884           27428 :         assert((size & chunksize_mask) == 0);
    2885                 : 
    2886           27428 :         ret = chunk_alloc_mmap(size, pagefile);
    2887           27428 :         if (ret != NULL) {
    2888           27410 :                 goto RETURN;
    2889                 :         }
    2890                 : 
    2891                 :         /* All strategies for allocation failed. */
    2892              18 :         ret = NULL;
    2893                 : RETURN:
    2894                 : #ifdef MALLOC_STATS
    2895           27428 :         if (ret != NULL)
    2896           27410 :                 stats_chunks.curchunks += (size / chunksize);
    2897           27428 :         if (stats_chunks.curchunks > stats_chunks.highchunks)
    2898           23539 :                 stats_chunks.highchunks = stats_chunks.curchunks;
    2899                 : #endif
    2900                 : 
    2901                 : #ifdef MALLOC_VALIDATE
    2902           27428 :         if (ret != NULL) {
    2903           27410 :                 if (malloc_rtree_set(chunk_rtree, (uintptr_t)ret, ret)) {
    2904               0 :                         chunk_dealloc(ret, size);
    2905               0 :                         return (NULL);
    2906                 :                 }
    2907                 :         }
    2908                 : #endif
    2909                 : 
    2910           27428 :         assert(CHUNK_ADDR2BASE(ret) == ret);
    2911           27428 :         return (ret);
    2912                 : }
    2913                 : 
    2914                 : static void
    2915            5814 : chunk_dealloc_mmap(void *chunk, size_t size)
    2916                 : {
    2917                 : 
    2918            5814 :         pages_unmap(chunk, size);
    2919            5814 : }
    2920                 : 
    2921                 : static void
    2922            5814 : chunk_dealloc(void *chunk, size_t size)
    2923                 : {
    2924                 : 
    2925            5814 :         assert(chunk != NULL);
    2926            5814 :         assert(CHUNK_ADDR2BASE(chunk) == chunk);
    2927            5814 :         assert(size != 0);
    2928            5814 :         assert((size & chunksize_mask) == 0);
    2929                 : 
    2930                 : #ifdef MALLOC_STATS
    2931            5814 :         stats_chunks.curchunks -= (size / chunksize);
    2932                 : #endif
    2933                 : #ifdef MALLOC_VALIDATE
    2934            5814 :         malloc_rtree_set(chunk_rtree, (uintptr_t)chunk, NULL);
    2935                 : #endif
    2936                 : 
    2937            5814 :         chunk_dealloc_mmap(chunk, size);
    2938            5814 : }
    2939                 : 
    2940                 : /*
    2941                 :  * End chunk management functions.
    2942                 :  */
    2943                 : /******************************************************************************/
    2944                 : /*
    2945                 :  * Begin arena.
    2946                 :  */
    2947                 : 
    2948                 : /*
    2949                 :  * Choose an arena based on a per-thread value (fast-path code, calls slow-path
    2950                 :  * code if necessary).
    2951                 :  */
    2952                 : static inline arena_t *
    2953        46372702 : choose_arena(void)
    2954                 : {
    2955                 :         arena_t *ret;
    2956                 : 
    2957                 :         /*
    2958                 :          * We can only use TLS if this is a PIC library, since for the static
    2959                 :          * library version, libc's malloc is used by TLS allocation, which
    2960                 :          * introduces a bootstrapping issue.
    2961                 :          */
    2962                 : #ifndef NO_TLS
    2963        46372702 :         if (isthreaded == false) {
    2964                 :             /* Avoid the overhead of TLS for single-threaded operation. */
    2965               0 :             return (arenas[0]);
    2966                 :         }
    2967                 : 
    2968                 : #  ifdef MOZ_MEMORY_WINDOWS
    2969                 :         ret = (arena_t*)TlsGetValue(tlsIndex);
    2970                 : #  else
    2971        46372702 :         ret = arenas_map;
    2972                 : #  endif
    2973                 : 
    2974        46372702 :         if (ret == NULL) {
    2975               5 :                 ret = choose_arena_hard();
    2976               5 :                 assert(ret != NULL);
    2977                 :         }
    2978                 : #else
    2979                 :         if (isthreaded && narenas > 1) {
    2980                 :                 unsigned long ind;
    2981                 : 
    2982                 :                 /*
    2983                 :                  * Hash _pthread_self() to one of the arenas.  There is a prime
    2984                 :                  * number of arenas, so this has a reasonable chance of
    2985                 :                  * working.  Even so, the hashing can be easily thwarted by
    2986                 :                  * inconvenient _pthread_self() values.  Without specific
    2987                 :                  * knowledge of how _pthread_self() calculates values, we can't
    2988                 :                  * easily do much better than this.
    2989                 :                  */
    2990                 :                 ind = (unsigned long) _pthread_self() % narenas;
    2991                 : 
    2992                 :                 /*
    2993                 :                  * Optimistially assume that arenas[ind] has been initialized.
    2994                 :                  * At worst, we find out that some other thread has already
    2995                 :                  * done so, after acquiring the lock in preparation.  Note that
    2996                 :                  * this lazy locking also has the effect of lazily forcing
    2997                 :                  * cache coherency; without the lock acquisition, there's no
    2998                 :                  * guarantee that modification of arenas[ind] by another thread
    2999                 :                  * would be seen on this CPU for an arbitrary amount of time.
    3000                 :                  *
    3001                 :                  * In general, this approach to modifying a synchronized value
    3002                 :                  * isn't a good idea, but in this case we only ever modify the
    3003                 :                  * value once, so things work out well.
    3004                 :                  */
    3005                 :                 ret = arenas[ind];
    3006                 :                 if (ret == NULL) {
    3007                 :                         /*
    3008                 :                          * Avoid races with another thread that may have already
    3009                 :                          * initialized arenas[ind].
    3010                 :                          */
    3011                 :                         malloc_spin_lock(&arenas_lock);
    3012                 :                         if (arenas[ind] == NULL)
    3013                 :                                 ret = arenas_extend((unsigned)ind);
    3014                 :                         else
    3015                 :                                 ret = arenas[ind];
    3016                 :                         malloc_spin_unlock(&arenas_lock);
    3017                 :                 }
    3018                 :         } else
    3019                 :                 ret = arenas[0];
    3020                 : #endif
    3021                 : 
    3022        46372702 :         assert(ret != NULL);
    3023        46372702 :         return (ret);
    3024                 : }
    3025                 : 
    3026                 : #ifndef NO_TLS
    3027                 : /*
    3028                 :  * Choose an arena based on a per-thread value (slow-path code only, called
    3029                 :  * only by choose_arena()).
    3030                 :  */
    3031                 : static arena_t *
    3032               5 : choose_arena_hard(void)
    3033                 : {
    3034                 :         arena_t *ret;
    3035                 : 
    3036               5 :         assert(isthreaded);
    3037                 : 
    3038                 : #ifdef MALLOC_BALANCE
    3039                 :         /* Seed the PRNG used for arena load balancing. */
    3040                 :         SPRN(balance, (uint32_t)(uintptr_t)(_pthread_self()));
    3041                 : #endif
    3042                 : 
    3043               5 :         if (narenas > 1) {
    3044                 : #ifdef MALLOC_BALANCE
    3045                 :                 unsigned ind;
    3046                 : 
    3047                 :                 ind = PRN(balance, narenas_2pow);
    3048                 :                 if ((ret = arenas[ind]) == NULL) {
    3049                 :                         malloc_spin_lock(&arenas_lock);
    3050                 :                         if ((ret = arenas[ind]) == NULL)
    3051                 :                                 ret = arenas_extend(ind);
    3052                 :                         malloc_spin_unlock(&arenas_lock);
    3053                 :                 }
    3054                 : #else
    3055               0 :                 malloc_spin_lock(&arenas_lock);
    3056               0 :                 if ((ret = arenas[next_arena]) == NULL)
    3057               0 :                         ret = arenas_extend(next_arena);
    3058               0 :                 next_arena = (next_arena + 1) % narenas;
    3059               0 :                 malloc_spin_unlock(&arenas_lock);
    3060                 : #endif
    3061                 :         } else
    3062               5 :                 ret = arenas[0];
    3063                 : 
    3064                 : #ifdef MOZ_MEMORY_WINDOWS
    3065                 :         TlsSetValue(tlsIndex, ret);
    3066                 : #else
    3067               5 :         arenas_map = ret;
    3068                 : #endif
    3069                 : 
    3070               5 :         return (ret);
    3071                 : }
    3072                 : #endif
    3073                 : 
    3074                 : static inline int
    3075          194091 : arena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b)
    3076                 : {
    3077          194091 :         uintptr_t a_chunk = (uintptr_t)a;
    3078          194091 :         uintptr_t b_chunk = (uintptr_t)b;
    3079                 : 
    3080          194091 :         assert(a != NULL);
    3081          194091 :         assert(b != NULL);
    3082                 : 
    3083          194091 :         return ((a_chunk > b_chunk) - (a_chunk < b_chunk));
    3084                 : }
    3085                 : 
    3086                 : /* Wrap red-black tree macros in functions. */
    3087          313285 : rb_wrap(static, arena_chunk_tree_dirty_, arena_chunk_tree_t,
    3088          313285 :     arena_chunk_t, link_dirty, arena_chunk_comp)
    3089                 : 
    3090                 : static inline int
    3091         1479607 : arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
    3092                 : {
    3093         1479607 :         uintptr_t a_mapelm = (uintptr_t)a;
    3094         1479607 :         uintptr_t b_mapelm = (uintptr_t)b;
    3095                 : 
    3096         1479607 :         assert(a != NULL);
    3097         1479607 :         assert(b != NULL);
    3098                 : 
    3099         1479607 :         return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm));
    3100                 : }
    3101                 : 
    3102                 : /* Wrap red-black tree macros in functions. */
    3103         4897235 : rb_wrap(static, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, link,
    3104         4897235 :     arena_run_comp)
    3105                 : 
    3106                 : static inline int
    3107        27108048 : arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
    3108                 : {
    3109                 :         int ret;
    3110        27108048 :         size_t a_size = a->bits & ~pagesize_mask;
    3111        27108048 :         size_t b_size = b->bits & ~pagesize_mask;
    3112                 : 
    3113        27108048 :         ret = (a_size > b_size) - (a_size < b_size);
    3114        27108048 :         if (ret == 0) {
    3115                 :                 uintptr_t a_mapelm, b_mapelm;
    3116                 : 
    3117         7525083 :                 if ((a->bits & CHUNK_MAP_KEY) == 0)
    3118         7407268 :                         a_mapelm = (uintptr_t)a;
    3119                 :                 else {
    3120                 :                         /*
    3121                 :                          * Treat keys as though they are lower than anything
    3122                 :                          * else.
    3123                 :                          */
    3124          117815 :                         a_mapelm = 0;
    3125                 :                 }
    3126         7525083 :                 b_mapelm = (uintptr_t)b;
    3127                 : 
    3128         7525083 :                 ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm);
    3129                 :         }
    3130                 : 
    3131        27108048 :         return (ret);
    3132                 : }
    3133                 : 
    3134                 : /* Wrap red-black tree macros in functions. */
    3135        17699161 : rb_wrap(static, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, link,
    3136        17699161 :     arena_avail_comp)
    3137                 : 
    3138                 : static inline void *
    3139        45735071 : arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
    3140                 : {
    3141                 :         void *ret;
    3142                 :         unsigned i, mask, bit, regind;
    3143                 : 
    3144        45735071 :         assert(run->magic == ARENA_RUN_MAGIC);
    3145        45735071 :         assert(run->regs_minelm < bin->regs_mask_nelms);
    3146                 : 
    3147                 :         /*
    3148                 :          * Move the first check outside the loop, so that run->regs_minelm can
    3149                 :          * be updated unconditionally, without the possibility of updating it
    3150                 :          * multiple times.
    3151                 :          */
    3152        45735071 :         i = run->regs_minelm;
    3153        45735071 :         mask = run->regs_mask[i];
    3154        45735071 :         if (mask != 0) {
    3155                 :                 /* Usable allocation found. */
    3156        44908552 :                 bit = ffs((int)mask) - 1;
    3157                 : 
    3158        44908552 :                 regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
    3159        44908552 :                 assert(regind < bin->nregs);
    3160        89817104 :                 ret = (void *)(((uintptr_t)run) + bin->reg0_offset
    3161        44908552 :                     + (bin->reg_size * regind));
    3162                 : 
    3163                 :                 /* Clear bit. */
    3164        44908552 :                 mask ^= (1U << bit);
    3165        44908552 :                 run->regs_mask[i] = mask;
    3166                 : 
    3167        44908552 :                 return (ret);
    3168                 :         }
    3169                 : 
    3170          979239 :         for (i++; i < bin->regs_mask_nelms; i++) {
    3171          979239 :                 mask = run->regs_mask[i];
    3172          979239 :                 if (mask != 0) {
    3173                 :                         /* Usable allocation found. */
    3174          826519 :                         bit = ffs((int)mask) - 1;
    3175                 : 
    3176          826519 :                         regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
    3177          826519 :                         assert(regind < bin->nregs);
    3178         1653038 :                         ret = (void *)(((uintptr_t)run) + bin->reg0_offset
    3179          826519 :                             + (bin->reg_size * regind));
    3180                 : 
    3181                 :                         /* Clear bit. */
    3182          826519 :                         mask ^= (1U << bit);
    3183          826519 :                         run->regs_mask[i] = mask;
    3184                 : 
    3185                 :                         /*
    3186                 :                          * Make a note that nothing before this element
    3187                 :                          * contains a free region.
    3188                 :                          */
    3189          826519 :                         run->regs_minelm = i; /* Low payoff: + (mask == 0); */
    3190                 : 
    3191          826519 :                         return (ret);
    3192                 :                 }
    3193                 :         }
    3194                 :         /* Not reached. */
    3195               0 :         assert(0);
    3196                 :         return (NULL);
    3197                 : }
    3198                 : 
    3199                 : static inline void
    3200        43659702 : arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size)
    3201                 : {
    3202                 :         /*
    3203                 :          * To divide by a number D that is not a power of two we multiply
    3204                 :          * by (2^21 / D) and then right shift by 21 positions.
    3205                 :          *
    3206                 :          *   X / D
    3207                 :          *
    3208                 :          * becomes
    3209                 :          *
    3210                 :          *   (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT
    3211                 :          */
    3212                 : #define SIZE_INV_SHIFT 21
    3213                 : #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s << QUANTUM_2POW_MIN)) + 1)
    3214                 :         static const unsigned size_invs[] = {
    3215                 :             SIZE_INV(3),
    3216                 :             SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
    3217                 :             SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
    3218                 :             SIZE_INV(12),SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
    3219                 :             SIZE_INV(16),SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
    3220                 :             SIZE_INV(20),SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
    3221                 :             SIZE_INV(24),SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
    3222                 :             SIZE_INV(28),SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
    3223                 : #if (QUANTUM_2POW_MIN < 4)
    3224                 :             ,
    3225                 :             SIZE_INV(32), SIZE_INV(33), SIZE_INV(34), SIZE_INV(35),
    3226                 :             SIZE_INV(36), SIZE_INV(37), SIZE_INV(38), SIZE_INV(39),
    3227                 :             SIZE_INV(40), SIZE_INV(41), SIZE_INV(42), SIZE_INV(43),
    3228                 :             SIZE_INV(44), SIZE_INV(45), SIZE_INV(46), SIZE_INV(47),
    3229                 :             SIZE_INV(48), SIZE_INV(49), SIZE_INV(50), SIZE_INV(51),
    3230                 :             SIZE_INV(52), SIZE_INV(53), SIZE_INV(54), SIZE_INV(55),
    3231                 :             SIZE_INV(56), SIZE_INV(57), SIZE_INV(58), SIZE_INV(59),
    3232                 :             SIZE_INV(60), SIZE_INV(61), SIZE_INV(62), SIZE_INV(63)
    3233                 : #endif
    3234                 :         };
    3235                 :         unsigned diff, regind, elm, bit;
    3236                 : 
    3237        43659702 :         assert(run->magic == ARENA_RUN_MAGIC);
    3238                 :         assert(((sizeof(size_invs)) / sizeof(unsigned)) + 3
    3239                 :             >= (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN));
    3240                 : 
    3241                 :         /*
    3242                 :          * Avoid doing division with a variable divisor if possible.  Using
    3243                 :          * actual division here can reduce allocator throughput by over 20%!
    3244                 :          */
    3245        43659702 :         diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset);
    3246        43659702 :         if ((size & (size - 1)) == 0) {
    3247                 :                 /*
    3248                 :                  * log2_table allows fast division of a power of two in the
    3249                 :                  * [1..128] range.
    3250                 :                  *
    3251                 :                  * (x / divisor) becomes (x >> log2_table[divisor - 1]).
    3252                 :                  */
    3253                 :                 static const unsigned char log2_table[] = {
    3254                 :                     0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4,
    3255                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
    3256                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    3257                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
    3258                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    3259                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    3260                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    3261                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
    3262                 :                 };
    3263                 : 
    3264        34733586 :                 if (size <= 128)
    3265        28967596 :                         regind = (diff >> log2_table[size - 1]);
    3266         5765990 :                 else if (size <= 32768)
    3267         5765990 :                         regind = diff >> (8 + log2_table[(size >> 8) - 1]);
    3268                 :                 else {
    3269                 :                         /*
    3270                 :                          * The run size is too large for us to use the lookup
    3271                 :                          * table.  Use real division.
    3272                 :                          */
    3273               0 :                         regind = diff / size;
    3274                 :                 }
    3275         8926116 :         } else if (size <= ((sizeof(size_invs) / sizeof(unsigned))
    3276                 :             << QUANTUM_2POW_MIN) + 2) {
    3277         8896621 :                 regind = size_invs[(size >> QUANTUM_2POW_MIN) - 3] * diff;
    3278         8896621 :                 regind >>= SIZE_INV_SHIFT;
    3279                 :         } else {
    3280                 :                 /*
    3281                 :                  * size_invs isn't large enough to handle this size class, so
    3282                 :                  * calculate regind using actual division.  This only happens
    3283                 :                  * if the user increases small_max via the 'S' runtime
    3284                 :                  * configuration option.
    3285                 :                  */
    3286           29495 :                 regind = diff / size;
    3287                 :         };
    3288        43659702 :         assert(diff == regind * size);
    3289        43659702 :         assert(regind < bin->nregs);
    3290                 : 
    3291        43659702 :         elm = regind >> (SIZEOF_INT_2POW + 3);
    3292        43659702 :         if (elm < run->regs_minelm)
    3293          624208 :                 run->regs_minelm = elm;
    3294        43659702 :         bit = regind - (elm << (SIZEOF_INT_2POW + 3));
    3295        43659702 :         assert((run->regs_mask[elm] & (1U << bit)) == 0);
    3296        43659702 :         run->regs_mask[elm] |= (1U << bit);
    3297                 : #undef SIZE_INV
    3298                 : #undef SIZE_INV_SHIFT
    3299        43659702 : }
    3300                 : 
    3301                 : static void
    3302         3698730 : arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
    3303                 :     bool zero)
    3304                 : {
    3305                 :         arena_chunk_t *chunk;
    3306                 :         size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i;
    3307                 : 
    3308         3698730 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
    3309         3698730 :         old_ndirty = chunk->ndirty;
    3310         3698730 :         run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk)
    3311                 :             >> pagesize_2pow);
    3312         3698730 :         total_pages = (chunk->map[run_ind].bits & ~pagesize_mask) >>
    3313                 :             pagesize_2pow;
    3314         3698730 :         need_pages = (size >> pagesize_2pow);
    3315         3698730 :         assert(need_pages > 0);
    3316         3698730 :         assert(need_pages <= total_pages);
    3317         3698730 :         rem_pages = total_pages - need_pages;
    3318                 : 
    3319         3698730 :         arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]);
    3320                 : 
    3321                 :         /* Keep track of trailing unused pages for later use. */
    3322         3698730 :         if (rem_pages > 0) {
    3323         7185680 :                 chunk->map[run_ind+need_pages].bits = (rem_pages <<
    3324         3592840 :                     pagesize_2pow) | (chunk->map[run_ind+need_pages].bits &
    3325                 :                     pagesize_mask);
    3326         7185680 :                 chunk->map[run_ind+total_pages-1].bits = (rem_pages <<
    3327         3592840 :                     pagesize_2pow) | (chunk->map[run_ind+total_pages-1].bits &
    3328                 :                     pagesize_mask);
    3329         3592840 :                 arena_avail_tree_insert(&arena->runs_avail,
    3330         3592840 :                     &chunk->map[run_ind+need_pages]);
    3331                 :         }
    3332                 : 
    3333        21780038 :         for (i = 0; i < need_pages; i++) {
    3334                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
    3335                 :                 /*
    3336                 :                  * Commit decommitted pages if necessary.  If a decommitted
    3337                 :                  * page is encountered, commit all needed adjacent decommitted
    3338                 :                  * pages in one operation, in order to reduce system call
    3339                 :                  * overhead.
    3340                 :                  */
    3341        18081308 :                 if (chunk->map[run_ind + i].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED) {
    3342                 :                         size_t j;
    3343                 : 
    3344                 :                         /*
    3345                 :                          * Advance i+j to just past the index of the last page
    3346                 :                          * to commit.  Clear CHUNK_MAP_DECOMMITTED and
    3347                 :                          * CHUNK_MAP_MADVISED along the way.
    3348                 :                          */
    3349        11216558 :                         for (j = 0; i + j < need_pages && (chunk->map[run_ind +
    3350         9221752 :                             i + j].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED); j++) {
    3351                 :                                 /* DECOMMITTED and MADVISED are mutually exclusive. */
    3352         4610876 :                                 assert(!(chunk->map[run_ind + i + j].bits & CHUNK_MAP_DECOMMITTED &&
    3353                 :                                          chunk->map[run_ind + i + j].bits & CHUNK_MAP_MADVISED));
    3354                 : 
    3355         4610876 :                                 chunk->map[run_ind + i + j].bits &=
    3356                 :                                     ~CHUNK_MAP_MADVISED_OR_DECOMMITTED;
    3357                 :                         }
    3358                 : 
    3359                 : #  ifdef MALLOC_DECOMMIT
    3360                 :                         pages_commit((void *)((uintptr_t)chunk + ((run_ind + i)
    3361                 :                             << pagesize_2pow)), (j << pagesize_2pow));
    3362                 : #    ifdef MALLOC_STATS
    3363                 :                         arena->stats.ncommit++;
    3364                 : #    endif
    3365                 : #  endif
    3366                 : 
    3367                 : #  ifdef MALLOC_STATS
    3368          997403 :                         arena->stats.committed += j;
    3369                 : #  endif
    3370                 : 
    3371                 : #  ifndef MALLOC_DECOMMIT
    3372                 :                 }
    3373                 : #  else
    3374                 :                 } else /* No need to zero since commit zeros. */
    3375                 : #  endif
    3376                 : 
    3377                 : #endif
    3378                 : 
    3379                 :                 /* Zero if necessary. */
    3380        18081308 :                 if (zero) {
    3381          790138 :                         if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED)
    3382                 :                             == 0) {
    3383                 :                                 VALGRIND_MALLOCLIKE_BLOCK((void *)((uintptr_t)
    3384                 :                                     chunk + ((run_ind + i) << pagesize_2pow)),
    3385                 :                                     pagesize, 0, false);
    3386          669225 :                                 memset((void *)((uintptr_t)chunk + ((run_ind
    3387          669225 :                                     + i) << pagesize_2pow)), 0, pagesize);
    3388                 :                                 VALGRIND_FREELIKE_BLOCK((void *)((uintptr_t)
    3389                 :                                     chunk + ((run_ind + i) << pagesize_2pow)),
    3390                 :                                     0);
    3391                 :                                 /* CHUNK_MAP_ZEROED is cleared below. */
    3392                 :                         }
    3393                 :                 }
    3394                 : 
    3395                 :                 /* Update dirty page accounting. */
    3396        18081308 :                 if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) {
    3397        13470432 :                         chunk->ndirty--;
    3398        13470432 :                         arena->ndirty--;
    3399                 :                         /* CHUNK_MAP_DIRTY is cleared below. */
    3400                 :                 }
    3401                 : 
    3402                 :                 /* Initialize the chunk map. */
    3403        18081308 :                 if (large) {
    3404         4539996 :                         chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE
    3405                 :                             | CHUNK_MAP_ALLOCATED;
    3406                 :                 } else {
    3407        27082624 :                         chunk->map[run_ind + i].bits = (size_t)run
    3408        13541312 :                             | CHUNK_MAP_ALLOCATED;
    3409                 :                 }
    3410                 :         }
    3411                 : 
    3412                 :         /*
    3413                 :          * Set the run size only in the first element for large runs.  This is
    3414                 :          * primarily a debugging aid, since the lack of size info for trailing
    3415                 :          * pages only matters if the application tries to operate on an
    3416                 :          * interior pointer.
    3417                 :          */
    3418         3698730 :         if (large)
    3419          646749 :                 chunk->map[run_ind].bits |= size;
    3420                 : 
    3421         3698730 :         if (chunk->ndirty == 0 && old_ndirty > 0)
    3422          133997 :                 arena_chunk_tree_dirty_remove(&arena->chunks_dirty, chunk);
    3423         3698730 : }
    3424                 : 
    3425                 : static void
    3426           22874 : arena_chunk_init(arena_t *arena, arena_chunk_t *chunk)
    3427                 : {
    3428                 :         arena_run_t *run;
    3429                 :         size_t i;
    3430                 : 
    3431                 :         VALGRIND_MALLOCLIKE_BLOCK(chunk, (arena_chunk_header_npages <<
    3432                 :             pagesize_2pow), 0, false);
    3433                 : #ifdef MALLOC_STATS
    3434           22874 :         arena->stats.mapped += chunksize;
    3435                 : #endif
    3436                 : 
    3437           22874 :         chunk->arena = arena;
    3438                 : 
    3439                 :         /*
    3440                 :          * Claim that no pages are in use, since the header is merely overhead.
    3441                 :          */
    3442           22874 :         chunk->ndirty = 0;
    3443                 : 
    3444                 :         /* Initialize the map to contain one maximal free untouched run. */
    3445           22874 :         run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages <<
    3446                 :             pagesize_2pow));
    3447           45748 :         for (i = 0; i < arena_chunk_header_npages; i++)
    3448           22874 :                 chunk->map[i].bits = 0;
    3449           22874 :         chunk->map[i].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
    3450         5809996 :         for (i++; i < chunk_npages-1; i++) {
    3451         5787122 :                 chunk->map[i].bits = CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
    3452                 :         }
    3453           22874 :         chunk->map[chunk_npages-1].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
    3454                 : 
    3455                 : #ifdef MALLOC_DECOMMIT
    3456                 :         /*
    3457                 :          * Start out decommitted, in order to force a closer correspondence
    3458                 :          * between dirty pages and committed untouched pages.
    3459                 :          */
    3460                 :         pages_decommit(run, arena_maxclass);
    3461                 : #  ifdef MALLOC_STATS
    3462                 :         arena->stats.ndecommit++;
    3463                 :         arena->stats.decommitted += (chunk_npages - arena_chunk_header_npages);
    3464                 : #  endif
    3465                 : #endif
    3466                 : #ifdef MALLOC_STATS
    3467           22874 :         arena->stats.committed += arena_chunk_header_npages;
    3468                 : #endif
    3469                 : 
    3470                 :         /* Insert the run into the runs_avail tree. */
    3471           22874 :         arena_avail_tree_insert(&arena->runs_avail,
    3472           22874 :             &chunk->map[arena_chunk_header_npages]);
    3473                 : 
    3474                 : #ifdef MALLOC_DOUBLE_PURGE
    3475                 :         LinkedList_Init(&chunk->chunks_madvised_elem);
    3476                 : #endif
    3477           22874 : }
    3478                 : 
    3479                 : static void
    3480           30235 : arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
    3481                 : {
    3482                 : 
    3483           30235 :         if (arena->spare != NULL) {
    3484            1278 :                 if (arena->spare->ndirty > 0) {
    3485            2392 :                         arena_chunk_tree_dirty_remove(
    3486            1196 :                             &chunk->arena->chunks_dirty, arena->spare);
    3487            1196 :                         arena->ndirty -= arena->spare->ndirty;
    3488                 : #ifdef MALLOC_STATS
    3489            1196 :                         arena->stats.committed -= arena->spare->ndirty;
    3490                 : #endif
    3491                 :                 }
    3492                 : 
    3493                 : #ifdef MALLOC_DOUBLE_PURGE
    3494                 :                 /* This is safe to do even if arena->spare is not in the list. */
    3495                 :                 LinkedList_Remove(&arena->spare->chunks_madvised_elem);
    3496                 : #endif
    3497                 : 
    3498                 :                 VALGRIND_FREELIKE_BLOCK(arena->spare, 0);
    3499            1278 :                 chunk_dealloc((void *)arena->spare, chunksize);
    3500                 : #ifdef MALLOC_STATS
    3501            1278 :                 arena->stats.mapped -= chunksize;
    3502            1278 :                 arena->stats.committed -= arena_chunk_header_npages;
    3503                 : #endif
    3504                 :         }
    3505                 : 
    3506                 :         /*
    3507                 :          * Remove run from runs_avail, so that the arena does not use it.
    3508                 :          * Dirty page flushing only uses the chunks_dirty tree, so leaving this
    3509                 :          * chunk in the chunks_* trees is sufficient for that purpose.
    3510                 :          */
    3511           30235 :         arena_avail_tree_remove(&arena->runs_avail,
    3512           30235 :             &chunk->map[arena_chunk_header_npages]);
    3513                 : 
    3514           30235 :         arena->spare = chunk;
    3515           30235 : }
    3516                 : 
    3517                 : static arena_run_t *
    3518         3689612 : arena_run_alloc(arena_t *arena, arena_bin_t *bin, size_t size, bool large,
    3519                 :     bool zero)
    3520                 : {
    3521                 :         arena_run_t *run;
    3522                 :         arena_chunk_map_t *mapelm, key;
    3523                 : 
    3524         3689612 :         assert(size <= arena_maxclass);
    3525         3689612 :         assert((size & pagesize_mask) == 0);
    3526                 : 
    3527                 :         /* Search the arena's chunks for the lowest best fit. */
    3528         3689612 :         key.bits = size | CHUNK_MAP_KEY;
    3529         3689612 :         mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key);
    3530         3689612 :         if (mapelm != NULL) {
    3531         3640705 :                 arena_chunk_t *chunk =
    3532         3640705 :                     (arena_chunk_t*)CHUNK_ADDR2BASE(mapelm);
    3533         7281410 :                 size_t pageind = ((uintptr_t)mapelm -
    3534         3640705 :                     (uintptr_t)chunk->map) /
    3535                 :                     sizeof(arena_chunk_map_t);
    3536                 : 
    3537         3640705 :                 run = (arena_run_t *)((uintptr_t)chunk + (pageind
    3538         3640705 :                     << pagesize_2pow));
    3539         3640705 :                 arena_run_split(arena, run, size, large, zero);
    3540         3640705 :                 return (run);
    3541                 :         }
    3542                 : 
    3543           48907 :         if (arena->spare != NULL) {
    3544                 :                 /* Use the spare. */
    3545           26033 :                 arena_chunk_t *chunk = arena->spare;
    3546           26033 :                 arena->spare = NULL;
    3547           26033 :                 run = (arena_run_t *)((uintptr_t)chunk +
    3548           26033 :                     (arena_chunk_header_npages << pagesize_2pow));
    3549                 :                 /* Insert the run into the runs_avail tree. */
    3550           26033 :                 arena_avail_tree_insert(&arena->runs_avail,
    3551           26033 :                     &chunk->map[arena_chunk_header_npages]);
    3552           26033 :                 arena_run_split(arena, run, size, large, zero);
    3553           26033 :                 return (run);
    3554                 :         }
    3555                 : 
    3556                 :         /*
    3557                 :          * No usable runs.  Create a new chunk from which to allocate
    3558                 :          * the run.
    3559                 :          */
    3560                 :         {
    3561           22874 :                 arena_chunk_t *chunk = (arena_chunk_t *)
    3562           22874 :                     chunk_alloc(chunksize, true, true);
    3563           22874 :                 if (chunk == NULL)
    3564               0 :                         return (NULL);
    3565                 : 
    3566           22874 :                 arena_chunk_init(arena, chunk);
    3567           22874 :                 run = (arena_run_t *)((uintptr_t)chunk +
    3568           22874 :                     (arena_chunk_header_npages << pagesize_2pow));
    3569                 :         }
    3570                 :         /* Update page map. */
    3571           22874 :         arena_run_split(arena, run, size, large, zero);
    3572           22874 :         return (run);
    3573                 : }
    3574                 : 
    3575                 : static void
    3576             165 : arena_purge(arena_t *arena)
    3577                 : {
    3578                 :         arena_chunk_t *chunk;
    3579                 :         size_t i, npages;
    3580                 : #ifdef MALLOC_DEBUG
    3581             165 :         size_t ndirty = 0;
    3582            1896 :         rb_foreach_begin(arena_chunk_t, link_dirty, &arena->chunks_dirty,
    3583                 :             chunk) {
    3584            1566 :                 ndirty += chunk->ndirty;
    3585            1566 :         } rb_foreach_end(arena_chunk_t, link_dirty, &arena->chunks_dirty, chunk)
    3586             165 :         assert(ndirty == arena->ndirty);
    3587                 : #endif
    3588             165 :         assert(arena->ndirty > opt_dirty_max);
    3589                 : 
    3590                 : #ifdef MALLOC_STATS
    3591             165 :         arena->stats.npurge++;
    3592                 : #endif
    3593                 : 
    3594                 :         /*
    3595                 :          * Iterate downward through chunks until enough dirty memory has been
    3596                 :          * purged.  Terminate as soon as possible in order to minimize the
    3597                 :          * number of system calls, even if a chunk has only been partially
    3598                 :          * purged.
    3599                 :          */
    3600            1244 :         while (arena->ndirty > (opt_dirty_max >> 1)) {
    3601                 : #ifdef MALLOC_DOUBLE_PURGE
    3602                 :                 bool madvised = false;
    3603                 : #endif
    3604             914 :                 chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty);
    3605             914 :                 assert(chunk != NULL);
    3606                 : 
    3607           73094 :                 for (i = chunk_npages - 1; chunk->ndirty > 0; i--) {
    3608           72345 :                         assert(i >= arena_chunk_header_npages);
    3609                 : 
    3610           72345 :                         if (chunk->map[i].bits & CHUNK_MAP_DIRTY) {
    3611                 : #ifdef MALLOC_DECOMMIT
    3612                 :                                 const size_t free_operation = CHUNK_MAP_DECOMMITTED;
    3613                 : #else
    3614            1770 :                                 const size_t free_operation = CHUNK_MAP_MADVISED;
    3615                 : #endif
    3616            1770 :                                 assert((chunk->map[i].bits &
    3617                 :                                         CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
    3618            1770 :                                 chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
    3619                 :                                 /* Find adjacent dirty run(s). */
    3620           97438 :                                 for (npages = 1;
    3621          190931 :                                      i > arena_chunk_header_npages &&
    3622           95263 :                                        (chunk->map[i - 1].bits & CHUNK_MAP_DIRTY);
    3623           93898 :                                      npages++) {
    3624           93898 :                                         i--;
    3625           93898 :                                         assert((chunk->map[i].bits &
    3626                 :                                                 CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
    3627           93898 :                                         chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
    3628                 :                                 }
    3629            1770 :                                 chunk->ndirty -= npages;
    3630            1770 :                                 arena->ndirty -= npages;
    3631                 : 
    3632                 : #ifdef MALLOC_DECOMMIT
    3633                 :                                 pages_decommit((void *)((uintptr_t)
    3634                 :                                     chunk + (i << pagesize_2pow)),
    3635                 :                                     (npages << pagesize_2pow));
    3636                 : #  ifdef MALLOC_STATS
    3637                 :                                 arena->stats.ndecommit++;
    3638                 :                                 arena->stats.decommitted += npages;
    3639                 : #  endif
    3640                 : #endif
    3641                 : #ifdef MALLOC_STATS
    3642            1770 :                                 arena->stats.committed -= npages;
    3643                 : #endif
    3644                 : 
    3645                 : #ifndef MALLOC_DECOMMIT
    3646            1770 :                                 madvise((void *)((uintptr_t)chunk + (i <<
    3647                 :                                     pagesize_2pow)), (npages << pagesize_2pow),
    3648                 :                                     MADV_FREE);
    3649                 : #  ifdef MALLOC_DOUBLE_PURGE
    3650                 :                                 madvised = true;
    3651                 : #  endif
    3652                 : #endif
    3653                 : #ifdef MALLOC_STATS
    3654            1770 :                                 arena->stats.nmadvise++;
    3655            1770 :                                 arena->stats.purged += npages;
    3656                 : #endif
    3657            1770 :                                 if (arena->ndirty <= (opt_dirty_max >> 1))
    3658             165 :                                         break;
    3659                 :                         }
    3660                 :                 }
    3661                 : 
    3662             914 :                 if (chunk->ndirty == 0) {
    3663             860 :                         arena_chunk_tree_dirty_remove(&arena->chunks_dirty,
    3664                 :                             chunk);
    3665                 :                 }
    3666                 : #ifdef MALLOC_DOUBLE_PURGE
    3667                 :                 if (madvised) {
    3668                 :                         /* The chunk might already be in the list, but this
    3669                 :                          * makes sure it's at the front. */
    3670                 :                         LinkedList_Remove(&chunk->chunks_madvised_elem);
    3671                 :                         LinkedList_InsertHead(&arena->chunks_madvised, &chunk->chunks_madvised_elem);
    3672                 :                 }
    3673                 : #endif
    3674                 :         }
    3675             165 : }
    3676                 : 
    3677                 : static void
    3678         3372371 : arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
    3679                 : {
    3680                 :         arena_chunk_t *chunk;
    3681                 :         size_t size, run_ind, run_pages;
    3682                 : 
    3683         3372371 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
    3684         3372371 :         run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk)
    3685                 :             >> pagesize_2pow);
    3686         3372371 :         assert(run_ind >= arena_chunk_header_npages);
    3687         3372371 :         assert(run_ind < chunk_npages);
    3688         3372371 :         if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0)
    3689          637739 :                 size = chunk->map[run_ind].bits & ~pagesize_mask;
    3690                 :         else
    3691         2734632 :                 size = run->bin->run_size;
    3692         3372371 :         run_pages = (size >> pagesize_2pow);
    3693                 : 
    3694                 :         /* Mark pages as unallocated in the chunk map. */
    3695         3372371 :         if (dirty) {
    3696                 :                 size_t i;
    3697                 : 
    3698        20482931 :                 for (i = 0; i < run_pages; i++) {
    3699        17110560 :                         assert((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY)
    3700                 :                             == 0);
    3701        17110560 :                         chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY;
    3702                 :                 }
    3703                 : 
    3704         3372371 :                 if (chunk->ndirty == 0) {
    3705          157649 :                         arena_chunk_tree_dirty_insert(&arena->chunks_dirty,
    3706                 :                             chunk);
    3707                 :                 }
    3708         3372371 :                 chunk->ndirty += run_pages;
    3709         3372371 :                 arena->ndirty += run_pages;
    3710                 :         } else {
    3711                 :                 size_t i;
    3712                 : 
    3713               0 :                 for (i = 0; i < run_pages; i++) {
    3714               0 :                         chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE |
    3715                 :                             CHUNK_MAP_ALLOCATED);
    3716                 :                 }
    3717                 :         }
    3718         3372371 :         chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
    3719                 :             pagesize_mask);
    3720         6744742 :         chunk->map[run_ind+run_pages-1].bits = size |
    3721         3372371 :             (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
    3722                 : 
    3723                 :         /* Try to coalesce forward. */
    3724         6731503 :         if (run_ind + run_pages < chunk_npages &&
    3725         3359132 :             (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) {
    3726         2801107 :                 size_t nrun_size = chunk->map[run_ind+run_pages].bits &
    3727                 :                     ~pagesize_mask;
    3728                 : 
    3729                 :                 /*
    3730                 :                  * Remove successor from runs_avail; the coalesced run is
    3731                 :                  * inserted later.
    3732                 :                  */
    3733         2801107 :                 arena_avail_tree_remove(&arena->runs_avail,
    3734         2801107 :                     &chunk->map[run_ind+run_pages]);
    3735                 : 
    3736         2801107 :                 size += nrun_size;
    3737         2801107 :                 run_pages = size >> pagesize_2pow;
    3738                 : 
    3739         2801107 :                 assert((chunk->map[run_ind+run_pages-1].bits & ~pagesize_mask)
    3740                 :                     == nrun_size);
    3741         2801107 :                 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
    3742                 :                     pagesize_mask);
    3743         5602214 :                 chunk->map[run_ind+run_pages-1].bits = size |
    3744         2801107 :                     (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
    3745                 :         }
    3746                 : 
    3747                 :         /* Try to coalesce backward. */
    3748         3372371 :         if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits &
    3749                 :             CHUNK_MAP_ALLOCATED) == 0) {
    3750          446690 :                 size_t prun_size = chunk->map[run_ind-1].bits & ~pagesize_mask;
    3751                 : 
    3752          446690 :                 run_ind -= prun_size >> pagesize_2pow;
    3753                 : 
    3754                 :                 /*
    3755                 :                  * Remove predecessor from runs_avail; the coalesced run is
    3756                 :                  * inserted later.
    3757                 :                  */
    3758          446690 :                 arena_avail_tree_remove(&arena->runs_avail,
    3759          446690 :                     &chunk->map[run_ind]);
    3760                 : 
    3761          446690 :                 size += prun_size;
    3762          446690 :                 run_pages = size >> pagesize_2pow;
    3763                 : 
    3764          446690 :                 assert((chunk->map[run_ind].bits & ~pagesize_mask) ==
    3765                 :                     prun_size);
    3766          446690 :                 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
    3767                 :                     pagesize_mask);
    3768          893380 :                 chunk->map[run_ind+run_pages-1].bits = size |
    3769          446690 :                     (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
    3770                 :         }
    3771                 : 
    3772                 :         /* Insert into runs_avail, now that coalescing is complete. */
    3773         3372371 :         arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]);
    3774                 : 
    3775                 :         /* Deallocate chunk if it is now completely unused. */
    3776         6744742 :         if ((chunk->map[arena_chunk_header_npages].bits & (~pagesize_mask |
    3777         3372371 :             CHUNK_MAP_ALLOCATED)) == arena_maxclass)
    3778           30235 :                 arena_chunk_dealloc(arena, chunk);
    3779                 : 
    3780                 :         /* Enforce opt_dirty_max. */
    3781         3372371 :         if (arena->ndirty > opt_dirty_max)
    3782             165 :                 arena_purge(arena);
    3783         3372371 : }
    3784                 : 
    3785                 : static void
    3786               0 : arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
    3787                 :     size_t oldsize, size_t newsize)
    3788                 : {
    3789               0 :         size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
    3790               0 :         size_t head_npages = (oldsize - newsize) >> pagesize_2pow;
    3791                 : 
    3792               0 :         assert(oldsize > newsize);
    3793                 : 
    3794                 :         /*
    3795                 :          * Update the chunk map so that arena_run_dalloc() can treat the
    3796                 :          * leading run as separately allocated.
    3797                 :          */
    3798               0 :         chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE |
    3799                 :             CHUNK_MAP_ALLOCATED;
    3800               0 :         chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE |
    3801                 :             CHUNK_MAP_ALLOCATED;
    3802                 : 
    3803               0 :         arena_run_dalloc(arena, run, false);
    3804               0 : }
    3805                 : 
    3806                 : static void
    3807             117 : arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
    3808                 :     size_t oldsize, size_t newsize, bool dirty)
    3809                 : {
    3810             117 :         size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
    3811             117 :         size_t npages = newsize >> pagesize_2pow;
    3812                 : 
    3813             117 :         assert(oldsize > newsize);
    3814                 : 
    3815                 :         /*
    3816                 :          * Update the chunk map so that arena_run_dalloc() can treat the
    3817                 :          * trailing run as separately allocated.
    3818                 :          */
    3819             117 :         chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE |
    3820                 :             CHUNK_MAP_ALLOCATED;
    3821             234 :         chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE
    3822             117 :             | CHUNK_MAP_ALLOCATED;
    3823                 : 
    3824             117 :         arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
    3825                 :             dirty);
    3826             117 : }
    3827                 : 
    3828                 : static arena_run_t *
    3829         3182856 : arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
    3830                 : {
    3831                 :         arena_chunk_map_t *mapelm;
    3832                 :         arena_run_t *run;
    3833                 :         unsigned i, remainder;
    3834                 : 
    3835                 :         /* Look for a usable run. */
    3836         3182856 :         mapelm = arena_run_tree_first(&bin->runs);
    3837         3182856 :         if (mapelm != NULL) {
    3838                 :                 /* run is guaranteed to have available space. */
    3839          130875 :                 arena_run_tree_remove(&bin->runs, mapelm);
    3840          130875 :                 run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
    3841                 : #ifdef MALLOC_STATS
    3842          130875 :                 bin->stats.reruns++;
    3843                 : #endif
    3844          130875 :                 return (run);
    3845                 :         }
    3846                 :         /* No existing runs have any space available. */
    3847                 : 
    3848                 :         /* Allocate a new run. */
    3849         3051981 :         run = arena_run_alloc(arena, bin, bin->run_size, false, false);
    3850         3051981 :         if (run == NULL)
    3851               0 :                 return (NULL);
    3852                 :         /*
    3853                 :          * Don't initialize if a race in arena_run_alloc() allowed an existing
    3854                 :          * run to become usable.
    3855                 :          */
    3856         3051981 :         if (run == bin->runcur)
    3857               0 :                 return (run);
    3858                 : 
    3859                 :         VALGRIND_MALLOCLIKE_BLOCK(run, sizeof(arena_run_t) + (sizeof(unsigned) *
    3860                 :             (bin->regs_mask_nelms - 1)), 0, false);
    3861                 : 
    3862                 :         /* Initialize run internals. */
    3863         3051981 :         run->bin = bin;
    3864                 : 
    3865         7742566 :         for (i = 0; i < bin->regs_mask_nelms - 1; i++)
    3866         4690585 :                 run->regs_mask[i] = UINT_MAX;
    3867         3051981 :         remainder = bin->nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1);
    3868         3051981 :         if (remainder == 0)
    3869               0 :                 run->regs_mask[i] = UINT_MAX;
    3870                 :         else {
    3871                 :                 /* The last element has spare bits that need to be unset. */
    3872         3051981 :                 run->regs_mask[i] = (UINT_MAX >> ((1U << (SIZEOF_INT_2POW + 3))
    3873         3051981 :                     - remainder));
    3874                 :         }
    3875                 : 
    3876         3051981 :         run->regs_minelm = 0;
    3877                 : 
    3878         3051981 :         run->nfree = bin->nregs;
    3879                 : #ifdef MALLOC_DEBUG
    3880         3051981 :         run->magic = ARENA_RUN_MAGIC;
    3881                 : #endif
    3882                 : 
    3883                 : #ifdef MALLOC_STATS
    3884         3051981 :         bin->stats.nruns++;
    3885         3051981 :         bin->stats.curruns++;
    3886         3051981 :         if (bin->stats.curruns > bin->stats.highruns)
    3887          678835 :                 bin->stats.highruns = bin->stats.curruns;
    3888                 : #endif
    3889         3051981 :         return (run);
    3890                 : }
    3891                 : 
    3892                 : /* bin->runcur must have space available before this function is called. */
    3893                 : static inline void *
    3894        45735071 : arena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run)
    3895                 : {
    3896                 :         void *ret;
    3897                 : 
    3898        45735071 :         assert(run->magic == ARENA_RUN_MAGIC);
    3899        45735071 :         assert(run->nfree > 0);
    3900                 : 
    3901        45735071 :         ret = arena_run_reg_alloc(run, bin);
    3902        45735071 :         assert(ret != NULL);
    3903        45735071 :         run->nfree--;
    3904                 : 
    3905        45735071 :         return (ret);
    3906                 : }
    3907                 : 
    3908                 : /* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */
    3909                 : static void *
    3910         3182856 : arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
    3911                 : {
    3912                 : 
    3913         3182856 :         bin->runcur = arena_bin_nonfull_run_get(arena, bin);
    3914         3182856 :         if (bin->runcur == NULL)
    3915               0 :                 return (NULL);
    3916         3182856 :         assert(bin->runcur->magic == ARENA_RUN_MAGIC);
    3917         3182856 :         assert(bin->runcur->nfree > 0);
    3918                 : 
    3919         3182856 :         return (arena_bin_malloc_easy(arena, bin, bin->runcur));
    3920                 : }
    3921                 : 
    3922                 : /*
    3923                 :  * Calculate bin->run_size such that it meets the following constraints:
    3924                 :  *
    3925                 :  *   *) bin->run_size >= min_run_size
    3926                 :  *   *) bin->run_size <= arena_maxclass
    3927                 :  *   *) bin->run_size <= RUN_MAX_SMALL
    3928                 :  *   *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
    3929                 :  *
    3930                 :  * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are
    3931                 :  * also calculated here, since these settings are all interdependent.
    3932                 :  */
    3933                 : static size_t
    3934          672084 : arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
    3935                 : {
    3936                 :         size_t try_run_size, good_run_size;
    3937                 :         unsigned good_nregs, good_mask_nelms, good_reg0_offset;
    3938                 :         unsigned try_nregs, try_mask_nelms, try_reg0_offset;
    3939                 : 
    3940          672084 :         assert(min_run_size >= pagesize);
    3941          672084 :         assert(min_run_size <= arena_maxclass);
    3942          672084 :         assert(min_run_size <= RUN_MAX_SMALL);
    3943                 : 
    3944                 :         /*
    3945                 :          * Calculate known-valid settings before entering the run_size
    3946                 :          * expansion loop, so that the first part of the loop always copies
    3947                 :          * valid settings.
    3948                 :          *
    3949                 :          * The do..while loop iteratively reduces the number of regions until
    3950                 :          * the run header and the regions no longer overlap.  A closed formula
    3951                 :          * would be quite messy, since there is an interdependency between the
    3952                 :          * header's mask length and the number of regions.
    3953                 :          */
    3954          672084 :         try_run_size = min_run_size;
    3955          672084 :         try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size)
    3956                 :             + 1; /* Counter-act try_nregs-- in loop. */
    3957                 :         do {
    3958         1381506 :                 try_nregs--;
    3959         2763012 :                 try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
    3960         1381506 :                     ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0);
    3961         1381506 :                 try_reg0_offset = try_run_size - (try_nregs * bin->reg_size);
    3962         1381506 :         } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1))
    3963         1381506 :             > try_reg0_offset);
    3964                 : 
    3965                 :         /* run_size expansion loop. */
    3966                 :         do {
    3967                 :                 /*
    3968                 :                  * Copy valid settings before trying more aggressive settings.
    3969                 :                  */
    3970          802767 :                 good_run_size = try_run_size;
    3971          802767 :                 good_nregs = try_nregs;
    3972          802767 :                 good_mask_nelms = try_mask_nelms;
    3973          802767 :                 good_reg0_offset = try_reg0_offset;
    3974                 : 
    3975                 :                 /* Try more aggressive settings. */
    3976          802767 :                 try_run_size += pagesize;
    3977         1605534 :                 try_nregs = ((try_run_size - sizeof(arena_run_t)) /
    3978          802767 :                     bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */
    3979                 :                 do {
    3980         2314956 :                         try_nregs--;
    3981         4629912 :                         try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
    3982         2314956 :                             ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ?
    3983                 :                             1 : 0);
    3984         2314956 :                         try_reg0_offset = try_run_size - (try_nregs *
    3985         2314956 :                             bin->reg_size);
    3986         2314956 :                 } while (sizeof(arena_run_t) + (sizeof(unsigned) *
    3987         2314956 :                     (try_mask_nelms - 1)) > try_reg0_offset);
    3988         1605534 :         } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL
    3989          746760 :             && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
    3990         1512189 :             && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
    3991                 : 
    3992          672084 :         assert(sizeof(arena_run_t) + (sizeof(unsigned) * (good_mask_nelms - 1))
    3993                 :             <= good_reg0_offset);
    3994          672084 :         assert((good_mask_nelms << (SIZEOF_INT_2POW + 3)) >= good_nregs);
    3995                 : 
    3996                 :         /* Copy final settings. */
    3997          672084 :         bin->run_size = good_run_size;
    3998          672084 :         bin->nregs = good_nregs;
    3999          672084 :         bin->regs_mask_nelms = good_mask_nelms;
    4000          672084 :         bin->reg0_offset = good_reg0_offset;
    4001                 : 
    4002          672084 :         return (good_run_size);
    4003                 : }
    4004                 : 
    4005                 : #ifdef MALLOC_BALANCE
    4006                 : static inline void
    4007                 : arena_lock_balance(arena_t *arena)
    4008                 : {
    4009                 :         unsigned contention;
    4010                 : 
    4011                 :         contention = malloc_spin_lock(&arena->lock);
    4012                 :         if (narenas > 1) {
    4013                 :                 /*
    4014                 :                  * Calculate the exponentially averaged contention for this
    4015                 :                  * arena.  Due to integer math always rounding down, this value
    4016                 :                  * decays somewhat faster then normal.
    4017                 :                  */
    4018                 :                 arena->contention = (((uint64_t)arena->contention
    4019                 :                     * (uint64_t)((1U << BALANCE_ALPHA_INV_2POW)-1))
    4020                 :                     + (uint64_t)contention) >> BALANCE_ALPHA_INV_2POW;
    4021                 :                 if (arena->contention >= opt_balance_threshold)
    4022                 :                         arena_lock_balance_hard(arena);
    4023                 :         }
    4024                 : }
    4025                 : 
    4026                 : static void
    4027                 : arena_lock_balance_hard(arena_t *arena)
    4028                 : {
    4029                 :         uint32_t ind;
    4030                 : 
    4031                 :         arena->contention = 0;
    4032                 : #ifdef MALLOC_STATS
    4033                 :         arena->stats.nbalance++;
    4034                 : #endif
    4035                 :         ind = PRN(balance, narenas_2pow);
    4036                 :         if (arenas[ind] != NULL) {
    4037                 : #ifdef MOZ_MEMORY_WINDOWS
    4038                 :                 TlsSetValue(tlsIndex, arenas[ind]);
    4039                 : #else
    4040                 :                 arenas_map = arenas[ind];
    4041                 : #endif
    4042                 :         } else {
    4043                 :                 malloc_spin_lock(&arenas_lock);
    4044                 :                 if (arenas[ind] != NULL) {
    4045                 : #ifdef MOZ_MEMORY_WINDOWS
    4046                 :                         TlsSetValue(tlsIndex, arenas[ind]);
    4047                 : #else
    4048                 :                         arenas_map = arenas[ind];
    4049                 : #endif
    4050                 :                 } else {
    4051                 : #ifdef MOZ_MEMORY_WINDOWS
    4052                 :                         TlsSetValue(tlsIndex, arenas_extend(ind));
    4053                 : #else
    4054                 :                         arenas_map = arenas_extend(ind);
    4055                 : #endif
    4056                 :                 }
    4057                 :                 malloc_spin_unlock(&arenas_lock);
    4058                 :         }
    4059                 : }
    4060                 : #endif
    4061                 : 
    4062                 : static inline void *
    4063        45735071 : arena_malloc_small(arena_t *arena, size_t size, bool zero)
    4064                 : {
    4065                 :         void *ret;
    4066                 :         arena_bin_t *bin;
    4067                 :         arena_run_t *run;
    4068                 : 
    4069        45735071 :         if (size < small_min) {
    4070                 :                 /* Tiny. */
    4071        11588699 :                 size = pow2_ceil(size);
    4072        11588699 :                 bin = &arena->bins[ffs((int)(size >> (TINY_MIN_2POW +
    4073                 :                     1)))];
    4074                 : #if (!defined(NDEBUG) || defined(MALLOC_STATS))
    4075                 :                 /*
    4076                 :                  * Bin calculation is always correct, but we may need
    4077                 :                  * to fix size for the purposes of assertions and/or
    4078                 :                  * stats accuracy.
    4079                 :                  */
    4080        11588699 :                 if (size < (1U << TINY_MIN_2POW))
    4081           39317 :                         size = (1U << TINY_MIN_2POW);
    4082                 : #endif
    4083        34146372 :         } else if (size <= small_max) {
    4084                 :                 /* Quantum-spaced. */
    4085        32199741 :                 size = QUANTUM_CEILING(size);
    4086        32199741 :                 bin = &arena->bins[ntbins + (size >> opt_quantum_2pow)
    4087                 :                     - 1];
    4088                 :         } else {
    4089                 :                 /* Sub-page. */
    4090         1946631 :                 size = pow2_ceil(size);
    4091         3893262 :                 bin = &arena->bins[ntbins + nqbins
    4092         1946631 :                     + (ffs((int)(size >> opt_small_max_2pow)) - 2)];
    4093                 :         }
    4094        45735071 :         assert(size == bin->reg_size);
    4095                 : 
    4096                 : #ifdef MALLOC_BALANCE
    4097                 :         arena_lock_balance(arena);
    4098                 : #else
    4099        45735071 :         malloc_spin_lock(&arena->lock);
    4100                 : #endif
    4101        45735071 :         if ((run = bin->runcur) != NULL && run->nfree > 0)
    4102        42552215 :                 ret = arena_bin_malloc_easy(arena, bin, run);
    4103                 :         else
    4104         3182856 :                 ret = arena_bin_malloc_hard(arena, bin);
    4105                 : 
    4106        45735071 :         if (ret == NULL) {
    4107               0 :                 malloc_spin_unlock(&arena->lock);
    4108               0 :                 return (NULL);
    4109                 :         }
    4110                 : 
    4111                 : #ifdef MALLOC_STATS
    4112        45735071 :         bin->stats.nrequests++;
    4113        45735071 :         arena->stats.nmalloc_small++;
    4114        45735071 :         arena->stats.allocated_small += size;
    4115                 : #endif
    4116        45735071 :         malloc_spin_unlock(&arena->lock);
    4117                 : 
    4118                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
    4119        45735071 :         if (zero == false) {
    4120                 : #ifdef MALLOC_FILL
    4121        43672870 :                 if (opt_junk)
    4122        43672870 :                         memset(ret, 0xa5, size);
    4123               0 :                 else if (opt_zero)
    4124               0 :                         memset(ret, 0, size);
    4125                 : #endif
    4126                 :         } else
    4127         2062201 :                 memset(ret, 0, size);
    4128                 : 
    4129        45735071 :         return (ret);
    4130                 : }
    4131                 : 
    4132                 : static void *
    4133          637631 : arena_malloc_large(arena_t *arena, size_t size, bool zero)
    4134                 : {
    4135                 :         void *ret;
    4136                 : 
    4137                 :         /* Large allocation. */
    4138          637631 :         size = PAGE_CEILING(size);
    4139                 : #ifdef MALLOC_BALANCE
    4140                 :         arena_lock_balance(arena);
    4141                 : #else
    4142          637631 :         malloc_spin_lock(&arena->lock);
    4143                 : #endif
    4144          637631 :         ret = (void *)arena_run_alloc(arena, NULL, size, true, zero);
    4145          637631 :         if (ret == NULL) {
    4146               0 :                 malloc_spin_unlock(&arena->lock);
    4147               0 :                 return (NULL);
    4148                 :         }
    4149                 : #ifdef MALLOC_STATS
    4150          637631 :         arena->stats.nmalloc_large++;
    4151          637631 :         arena->stats.allocated_large += size;
    4152                 : #endif
    4153          637631 :         malloc_spin_unlock(&arena->lock);
    4154                 : 
    4155                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
    4156          637631 :         if (zero == false) {
    4157                 : #ifdef MALLOC_FILL
    4158          443449 :                 if (opt_junk)
    4159          443449 :                         memset(ret, 0xa5, size);
    4160               0 :                 else if (opt_zero)
    4161               0 :                         memset(ret, 0, size);
    4162                 : #endif
    4163                 :         }
    4164                 : 
    4165          637631 :         return (ret);
    4166                 : }
    4167                 : 
    4168                 : static inline void *
    4169        46372702 : arena_malloc(arena_t *arena, size_t size, bool zero)
    4170                 : {
    4171                 : 
    4172        46372702 :         assert(arena != NULL);
    4173        46372702 :         assert(arena->magic == ARENA_MAGIC);
    4174        46372702 :         assert(size != 0);
    4175        46372702 :         assert(QUANTUM_CEILING(size) <= arena_maxclass);
    4176                 : 
    4177        46372702 :         if (size <= bin_maxclass) {
    4178        45735071 :                 return (arena_malloc_small(arena, size, zero));
    4179                 :         } else
    4180          637631 :                 return (arena_malloc_large(arena, size, zero));
    4181                 : }
    4182                 : 
    4183                 : static inline void *
    4184        41858280 : imalloc(size_t size)
    4185                 : {
    4186                 : 
    4187        41858280 :         assert(size != 0);
    4188                 : 
    4189        41858280 :         if (size <= arena_maxclass)
    4190        41855450 :                 return (arena_malloc(choose_arena(), size, false));
    4191                 :         else
    4192            2830 :                 return (huge_malloc(size, false));
    4193                 : }
    4194                 : 
    4195                 : static inline void *
    4196         2256419 : icalloc(size_t size)
    4197                 : {
    4198                 : 
    4199         2256419 :         if (size <= arena_maxclass)
    4200         2256383 :                 return (arena_malloc(choose_arena(), size, true));
    4201                 :         else
    4202              36 :                 return (huge_malloc(size, true));
    4203                 : }
    4204                 : 
    4205                 : /* Only handles large allocations that require more than page alignment. */
    4206                 : static void *
    4207               0 : arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
    4208                 : {
    4209                 :         void *ret;
    4210                 :         size_t offset;
    4211                 :         arena_chunk_t *chunk;
    4212                 : 
    4213               0 :         assert((size & pagesize_mask) == 0);
    4214               0 :         assert((alignment & pagesize_mask) == 0);
    4215                 : 
    4216                 : #ifdef MALLOC_BALANCE
    4217                 :         arena_lock_balance(arena);
    4218                 : #else
    4219               0 :         malloc_spin_lock(&arena->lock);
    4220                 : #endif
    4221               0 :         ret = (void *)arena_run_alloc(arena, NULL, alloc_size, true, false);
    4222               0 :         if (ret == NULL) {
    4223               0 :                 malloc_spin_unlock(&arena->lock);
    4224               0 :                 return (NULL);
    4225                 :         }
    4226                 : 
    4227               0 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
    4228                 : 
    4229               0 :         offset = (uintptr_t)ret & (alignment - 1);
    4230               0 :         assert((offset & pagesize_mask) == 0);
    4231               0 :         assert(offset < alloc_size);
    4232               0 :         if (offset == 0)
    4233               0 :                 arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, alloc_size, size, false);
    4234                 :         else {
    4235                 :                 size_t leadsize, trailsize;
    4236                 : 
    4237               0 :                 leadsize = alignment - offset;
    4238               0 :                 if (leadsize > 0) {
    4239               0 :                         arena_run_trim_head(arena, chunk, (arena_run_t*)ret, alloc_size,
    4240                 :                             alloc_size - leadsize);
    4241               0 :                         ret = (void *)((uintptr_t)ret + leadsize);
    4242                 :                 }
    4243                 : 
    4244               0 :                 trailsize = alloc_size - leadsize - size;
    4245               0 :                 if (trailsize != 0) {
    4246                 :                         /* Trim trailing space. */
    4247               0 :                         assert(trailsize < alloc_size);
    4248               0 :                         arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, size + trailsize,
    4249                 :                             size, false);
    4250                 :                 }
    4251                 :         }
    4252                 : 
    4253                 : #ifdef MALLOC_STATS
    4254               0 :         arena->stats.nmalloc_large++;
    4255               0 :         arena->stats.allocated_large += size;
    4256                 : #endif
    4257               0 :         malloc_spin_unlock(&arena->lock);
    4258                 : 
    4259                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
    4260                 : #ifdef MALLOC_FILL
    4261               0 :         if (opt_junk)
    4262               0 :                 memset(ret, 0xa5, size);
    4263               0 :         else if (opt_zero)
    4264               0 :                 memset(ret, 0, size);
    4265                 : #endif
    4266               0 :         return (ret);
    4267                 : }
    4268                 : 
    4269                 : static inline void *
    4270               0 : ipalloc(size_t alignment, size_t size)
    4271                 : {
    4272                 :         void *ret;
    4273                 :         size_t ceil_size;
    4274                 : 
    4275                 :         /*
    4276                 :          * Round size up to the nearest multiple of alignment.
    4277                 :          *
    4278                 :          * This done, we can take advantage of the fact that for each small
    4279                 :          * size class, every object is aligned at the smallest power of two
    4280                 :          * that is non-zero in the base two representation of the size.  For
    4281                 :          * example:
    4282                 :          *
    4283                 :          *   Size |   Base 2 | Minimum alignment
    4284                 :          *   -----+----------+------------------
    4285                 :          *     96 |  1100000 |  32
    4286                 :          *    144 | 10100000 |  32
    4287                 :          *    192 | 11000000 |  64
    4288                 :          *
    4289                 :          * Depending on runtime settings, it is possible that arena_malloc()
    4290                 :          * will further round up to a power of two, but that never causes
    4291                 :          * correctness issues.
    4292                 :          */
    4293               0 :         ceil_size = (size + (alignment - 1)) & (-alignment);
    4294                 :         /*
    4295                 :          * (ceil_size < size) protects against the combination of maximal
    4296                 :          * alignment and size greater than maximal alignment.
    4297                 :          */
    4298               0 :         if (ceil_size < size) {
    4299                 :                 /* size_t overflow. */
    4300               0 :                 return (NULL);
    4301                 :         }
    4302                 : 
    4303               0 :         if (ceil_size <= pagesize || (alignment <= pagesize
    4304               0 :             && ceil_size <= arena_maxclass))
    4305               0 :                 ret = arena_malloc(choose_arena(), ceil_size, false);
    4306                 :         else {
    4307                 :                 size_t run_size;
    4308                 : 
    4309                 :                 /*
    4310                 :                  * We can't achieve sub-page alignment, so round up alignment
    4311                 :                  * permanently; it makes later calculations simpler.
    4312                 :                  */
    4313               0 :                 alignment = PAGE_CEILING(alignment);
    4314               0 :                 ceil_size = PAGE_CEILING(size);
    4315                 :                 /*
    4316                 :                  * (ceil_size < size) protects against very large sizes within
    4317                 :                  * pagesize of SIZE_T_MAX.
    4318                 :                  *
    4319                 :                  * (ceil_size + alignment < ceil_size) protects against the
    4320                 :                  * combination of maximal alignment and ceil_size large enough
    4321                 :                  * to cause overflow.  This is similar to the first overflow
    4322                 :                  * check above, but it needs to be repeated due to the new
    4323                 :                  * ceil_size value, which may now be *equal* to maximal
    4324                 :                  * alignment, whereas before we only detected overflow if the
    4325                 :                  * original size was *greater* than maximal alignment.
    4326                 :                  */
    4327               0 :                 if (ceil_size < size || ceil_size + alignment < ceil_size) {
    4328                 :                         /* size_t overflow. */
    4329               0 :                         return (NULL);
    4330                 :                 }
    4331                 : 
    4332                 :                 /*
    4333                 :                  * Calculate the size of the over-size run that arena_palloc()
    4334                 :                  * would need to allocate in order to guarantee the alignment.
    4335                 :                  */
    4336               0 :                 if (ceil_size >= alignment)
    4337               0 :                         run_size = ceil_size + alignment - pagesize;
    4338                 :                 else {
    4339                 :                         /*
    4340                 :                          * It is possible that (alignment << 1) will cause
    4341                 :                          * overflow, but it doesn't matter because we also
    4342                 :                          * subtract pagesize, which in the case of overflow
    4343                 :                          * leaves us with a very large run_size.  That causes
    4344                 :                          * the first conditional below to fail, which means
    4345                 :                          * that the bogus run_size value never gets used for
    4346                 :                          * anything important.
    4347                 :                          */
    4348               0 :                         run_size = (alignment << 1) - pagesize;
    4349                 :                 }
    4350                 : 
    4351               0 :                 if (run_size <= arena_maxclass) {
    4352               0 :                         ret = arena_palloc(choose_arena(), alignment, ceil_size,
    4353                 :                             run_size);
    4354               0 :                 } else if (alignment <= chunksize)
    4355               0 :                         ret = huge_malloc(ceil_size, false);
    4356                 :                 else
    4357               0 :                         ret = huge_palloc(alignment, ceil_size);
    4358                 :         }
    4359                 : 
    4360               0 :         assert(((uintptr_t)ret & (alignment - 1)) == 0);
    4361               0 :         return (ret);
    4362                 : }
    4363                 : 
    4364                 : /* Return the size of the allocation pointed to by ptr. */
    4365                 : static size_t
    4366         2300043 : arena_salloc(const void *ptr)
    4367                 : {
    4368                 :         size_t ret;
    4369                 :         arena_chunk_t *chunk;
    4370                 :         size_t pageind, mapbits;
    4371                 : 
    4372         2300043 :         assert(ptr != NULL);
    4373         2300043 :         assert(CHUNK_ADDR2BASE(ptr) != ptr);
    4374                 : 
    4375         2300043 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
    4376         2300043 :         pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow);
    4377         2300043 :         mapbits = chunk->map[pageind].bits;
    4378         2300043 :         assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
    4379         2300043 :         if ((mapbits & CHUNK_MAP_LARGE) == 0) {
    4380         2271520 :                 arena_run_t *run = (arena_run_t *)(mapbits & ~pagesize_mask);
    4381         2271520 :                 assert(run->magic == ARENA_RUN_MAGIC);
    4382         2271520 :                 ret = run->bin->reg_size;
    4383                 :         } else {
    4384           28523 :                 ret = mapbits & ~pagesize_mask;
    4385           28523 :                 assert(ret != 0);
    4386                 :         }
    4387                 : 
    4388         2300043 :         return (ret);
    4389                 : }
    4390                 : 
    4391                 : #if (defined(MALLOC_VALIDATE) || defined(MOZ_MEMORY_DARWIN))
    4392                 : /*
    4393                 :  * Validate ptr before assuming that it points to an allocation.  Currently,
    4394                 :  * the following validation is performed:
    4395                 :  *
    4396                 :  * + Check that ptr is not NULL.
    4397                 :  *
    4398                 :  * + Check that ptr lies within a mapped chunk.
    4399                 :  */
    4400                 : static inline size_t
    4401               0 : isalloc_validate(const void *ptr)
    4402                 : {
    4403                 :         arena_chunk_t *chunk;
    4404                 : 
    4405               0 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
    4406               0 :         if (chunk == NULL)
    4407               0 :                 return (0);
    4408                 : 
    4409               0 :         if (malloc_rtree_get(chunk_rtree, (uintptr_t)chunk) == NULL)
    4410               0 :                 return (0);
    4411                 : 
    4412               0 :         if (chunk != ptr) {
    4413               0 :                 assert(chunk->arena->magic == ARENA_MAGIC);
    4414               0 :                 return (arena_salloc(ptr));
    4415                 :         } else {
    4416                 :                 size_t ret;
    4417                 :                 extent_node_t *node;
    4418                 :                 extent_node_t key;
    4419                 : 
    4420                 :                 /* Chunk. */
    4421               0 :                 key.addr = (void *)chunk;
    4422               0 :                 malloc_mutex_lock(&huge_mtx);
    4423               0 :                 node = extent_tree_ad_search(&huge, &key);
    4424               0 :                 if (node != NULL)
    4425               0 :                         ret = node->size;
    4426                 :                 else
    4427               0 :                         ret = 0;
    4428               0 :                 malloc_mutex_unlock(&huge_mtx);
    4429               0 :                 return (ret);
    4430                 :         }
    4431                 : }
    4432                 : #endif
    4433                 : 
    4434                 : static inline size_t
    4435         2301663 : isalloc(const void *ptr)
    4436                 : {
    4437                 :         size_t ret;
    4438                 :         arena_chunk_t *chunk;
    4439                 : 
    4440         2301663 :         assert(ptr != NULL);
    4441                 : 
    4442         2301663 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
    4443         2301663 :         if (chunk != ptr) {
    4444                 :                 /* Region. */
    4445         2300043 :                 assert(chunk->arena->magic == ARENA_MAGIC);
    4446                 : 
    4447         2300043 :                 ret = arena_salloc(ptr);
    4448                 :         } else {
    4449                 :                 extent_node_t *node, key;
    4450                 : 
    4451                 :                 /* Chunk (huge allocation). */
    4452                 : 
    4453            1620 :                 malloc_mutex_lock(&huge_mtx);
    4454                 : 
    4455                 :                 /* Extract from tree of huge allocations. */
    4456            1620 :                 key.addr = __DECONST(void *, ptr);
    4457            1620 :                 node = extent_tree_ad_search(&huge, &key);
    4458            1620 :                 assert(node != NULL);
    4459                 : 
    4460            1620 :                 ret = node->size;
    4461                 : 
    4462            1620 :                 malloc_mutex_unlock(&huge_mtx);
    4463                 :         }
    4464                 : 
    4465         2301663 :         return (ret);
    4466                 : }
    4467                 : 
    4468                 : static inline void
    4469        43659702 : arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
    4470                 :     arena_chunk_map_t *mapelm)
    4471                 : {
    4472                 :         arena_run_t *run;
    4473                 :         arena_bin_t *bin;
    4474                 :         size_t size;
    4475                 : 
    4476        43659702 :         run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
    4477        43659702 :         assert(run->magic == ARENA_RUN_MAGIC);
    4478        43659702 :         bin = run->bin;
    4479        43659702 :         size = bin->reg_size;
    4480                 : 
    4481                 : #ifdef MALLOC_FILL
    4482        43659702 :         if (opt_junk)
    4483        43659702 :                 memset(ptr, 0x5a, size);
    4484                 : #endif
    4485                 : 
    4486        43659702 :         arena_run_reg_dalloc(run, bin, ptr, size);
    4487        43659702 :         run->nfree++;
    4488                 : 
    4489        43659702 :         if (run->nfree == bin->nregs) {
    4490                 :                 /* Deallocate run. */
    4491         2734632 :                 if (run == bin->runcur)
    4492         2572218 :                         bin->runcur = NULL;
    4493          162414 :                 else if (bin->nregs != 1) {
    4494          324828 :                         size_t run_pageind = (((uintptr_t)run -
    4495          162414 :                             (uintptr_t)chunk)) >> pagesize_2pow;
    4496          162414 :                         arena_chunk_map_t *run_mapelm =
    4497          162414 :                             &chunk->map[run_pageind];
    4498                 :                         /*
    4499                 :                          * This block's conditional is necessary because if the
    4500                 :                          * run only contains one region, then it never gets
    4501                 :                          * inserted into the non-full runs tree.
    4502                 :                          */
    4503          162414 :                         assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
    4504                 :                             run_mapelm);
    4505          162414 :                         arena_run_tree_remove(&bin->runs, run_mapelm);
    4506                 :                 }
    4507                 : #ifdef MALLOC_DEBUG
    4508         2734632 :                 run->magic = 0;
    4509                 : #endif
    4510                 :                 VALGRIND_FREELIKE_BLOCK(run, 0);
    4511         2734632 :                 arena_run_dalloc(arena, run, true);
    4512                 : #ifdef MALLOC_STATS
    4513         2734632 :                 bin->stats.curruns--;
    4514                 : #endif
    4515        40925070 :         } else if (run->nfree == 1 && run != bin->runcur) {
    4516                 :                 /*
    4517                 :                  * Make sure that bin->runcur always refers to the lowest
    4518                 :                  * non-full run, if one exists.
    4519                 :                  */
    4520          300701 :                 if (bin->runcur == NULL)
    4521            5398 :                         bin->runcur = run;
    4522          295303 :                 else if ((uintptr_t)run < (uintptr_t)bin->runcur) {
    4523                 :                         /* Switch runcur. */
    4524          212684 :                         if (bin->runcur->nfree > 0) {
    4525          210677 :                                 arena_chunk_t *runcur_chunk =
    4526          210677 :                                     (arena_chunk_t*)CHUNK_ADDR2BASE(bin->runcur);
    4527          210677 :                                 size_t runcur_pageind =
    4528          421354 :                                     (((uintptr_t)bin->runcur -
    4529          210677 :                                     (uintptr_t)runcur_chunk)) >> pagesize_2pow;
    4530          210677 :                                 arena_chunk_map_t *runcur_mapelm =
    4531          210677 :                                     &runcur_chunk->map[runcur_pageind];
    4532                 : 
    4533                 :                                 /* Insert runcur. */
    4534          210677 :                                 assert(arena_run_tree_search(&bin->runs,
    4535                 :                                     runcur_mapelm) == NULL);
    4536          210677 :                                 arena_run_tree_insert(&bin->runs,
    4537                 :                                     runcur_mapelm);
    4538                 :                         }
    4539          212684 :                         bin->runcur = run;
    4540                 :                 } else {
    4541          165238 :                         size_t run_pageind = (((uintptr_t)run -
    4542           82619 :                             (uintptr_t)chunk)) >> pagesize_2pow;
    4543           82619 :                         arena_chunk_map_t *run_mapelm =
    4544           82619 :                             &chunk->map[run_pageind];
    4545                 : 
    4546           82619 :                         assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
    4547                 :                             NULL);
    4548           82619 :                         arena_run_tree_insert(&bin->runs, run_mapelm);
    4549                 :                 }
    4550                 :         }
    4551                 : #ifdef MALLOC_STATS
    4552        43659702 :         arena->stats.allocated_small -= size;
    4553        43659702 :         arena->stats.ndalloc_small++;
    4554                 : #endif
    4555        43659702 : }
    4556                 : 
    4557                 : static void
    4558          637622 : arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
    4559                 : {
    4560                 :         /* Large allocation. */
    4561          637622 :         malloc_spin_lock(&arena->lock);
    4562                 : 
    4563                 : #ifdef MALLOC_FILL
    4564                 : #ifndef MALLOC_STATS
    4565                 :         if (opt_junk)
    4566                 : #endif
    4567                 : #endif
    4568                 :         {
    4569          637622 :                 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
    4570                 :                     pagesize_2pow;
    4571          637622 :                 size_t size = chunk->map[pageind].bits & ~pagesize_mask;
    4572                 : 
    4573                 : #ifdef MALLOC_FILL
    4574                 : #ifdef MALLOC_STATS
    4575          637622 :                 if (opt_junk)
    4576                 : #endif
    4577          637622 :                         memset(ptr, 0x5a, size);
    4578                 : #endif
    4579                 : #ifdef MALLOC_STATS
    4580          637622 :                 arena->stats.allocated_large -= size;
    4581                 : #endif
    4582                 :         }
    4583                 : #ifdef MALLOC_STATS
    4584          637622 :         arena->stats.ndalloc_large++;
    4585                 : #endif
    4586                 : 
    4587          637622 :         arena_run_dalloc(arena, (arena_run_t *)ptr, true);
    4588          637622 :         malloc_spin_unlock(&arena->lock);
    4589          637622 : }
    4590                 : 
    4591                 : static inline void
    4592        44297318 : arena_dalloc(void *ptr, size_t offset)
    4593                 : {
    4594                 :         arena_chunk_t *chunk;
    4595                 :         arena_t *arena;
    4596                 :         size_t pageind;
    4597                 :         arena_chunk_map_t *mapelm;
    4598                 : 
    4599        44297318 :         assert(ptr != NULL);
    4600        44297318 :         assert(offset != 0);
    4601        44297318 :         assert(CHUNK_ADDR2OFFSET(ptr) == offset);
    4602                 : 
    4603        44297318 :         chunk = (arena_chunk_t *) ((uintptr_t)ptr - offset);
    4604        44297318 :         arena = chunk->arena;
    4605        44297318 :         assert(arena != NULL);
    4606        44297318 :         assert(arena->magic == ARENA_MAGIC);
    4607                 : 
    4608        44297318 :         pageind = offset >> pagesize_2pow;
    4609        44297318 :         mapelm = &chunk->map[pageind];
    4610        44297318 :         assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
    4611        44297318 :         if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
    4612                 :                 /* Small allocation. */
    4613        43659696 :                 malloc_spin_lock(&arena->lock);
    4614        43659702 :                 arena_dalloc_small(arena, chunk, ptr, mapelm);
    4615        43659702 :                 malloc_spin_unlock(&arena->lock);
    4616                 :         } else
    4617          637622 :                 arena_dalloc_large(arena, chunk, ptr);
    4618                 :         VALGRIND_FREELIKE_BLOCK(ptr, 0);
    4619        44297323 : }
    4620                 : 
    4621                 : static inline void
    4622         2262557 : idalloc(void *ptr)
    4623                 : {
    4624                 :         size_t offset;
    4625                 : 
    4626         2262557 :         assert(ptr != NULL);
    4627                 : 
    4628         2262557 :         offset = CHUNK_ADDR2OFFSET(ptr);
    4629         2262557 :         if (offset != 0)
    4630         2260937 :                 arena_dalloc(ptr, offset);
    4631                 :         else
    4632            1620 :                 huge_dalloc(ptr);
    4633         2262557 : }
    4634                 : 
    4635                 : static void
    4636             117 : arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
    4637                 :     size_t size, size_t oldsize)
    4638                 : {
    4639                 : 
    4640             117 :         assert(size < oldsize);
    4641                 : 
    4642                 :         /*
    4643                 :          * Shrink the run, and make trailing pages available for other
    4644                 :          * allocations.
    4645                 :          */
    4646                 : #ifdef MALLOC_BALANCE
    4647                 :         arena_lock_balance(arena);
    4648                 : #else
    4649             117 :         malloc_spin_lock(&arena->lock);
    4650                 : #endif
    4651             117 :         arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size,
    4652                 :             true);
    4653                 : #ifdef MALLOC_STATS
    4654             117 :         arena->stats.allocated_large -= oldsize - size;
    4655                 : #endif
    4656             117 :         malloc_spin_unlock(&arena->lock);
    4657             117 : }
    4658                 : 
    4659                 : static bool
    4660           28113 : arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
    4661                 :     size_t size, size_t oldsize)
    4662                 : {
    4663           28113 :         size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow;
    4664           28113 :         size_t npages = oldsize >> pagesize_2pow;
    4665                 : 
    4666           28113 :         assert(oldsize == (chunk->map[pageind].bits & ~pagesize_mask));
    4667                 : 
    4668                 :         /* Try to extend the run. */
    4669           28113 :         assert(size > oldsize);
    4670                 : #ifdef MALLOC_BALANCE
    4671                 :         arena_lock_balance(arena);
    4672                 : #else
    4673           28113 :         malloc_spin_lock(&arena->lock);
    4674                 : #endif
    4675           51977 :         if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits
    4676           44938 :             & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits &
    4677           10537 :             ~pagesize_mask) >= size - oldsize) {
    4678                 :                 /*
    4679                 :                  * The next run is available and sufficiently large.  Split the
    4680                 :                  * following run, then merge the first part with the existing
    4681                 :                  * allocation.
    4682                 :                  */
    4683           18236 :                 arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk +
    4684            9118 :                     ((pageind+npages) << pagesize_2pow)), size - oldsize, true,
    4685                 :                     false);
    4686                 : 
    4687            9118 :                 chunk->map[pageind].bits = size | CHUNK_MAP_LARGE |
    4688                 :                     CHUNK_MAP_ALLOCATED;
    4689            9118 :                 chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE |
    4690                 :                     CHUNK_MAP_ALLOCATED;
    4691                 : 
    4692                 : #ifdef MALLOC_STATS
    4693            9118 :                 arena->stats.allocated_large += size - oldsize;
    4694                 : #endif
    4695            9118 :                 malloc_spin_unlock(&arena->lock);
    4696            9118 :                 return (false);
    4697                 :         }
    4698           18995 :         malloc_spin_unlock(&arena->lock);
    4699                 : 
    4700           18995 :         return (true);
    4701                 : }
    4702                 : 
    4703                 : /*
    4704                 :  * Try to resize a large allocation, in order to avoid copying.  This will
    4705                 :  * always fail if growing an object, and the following run is already in use.
    4706                 :  */
    4707                 : static bool
    4708           28338 : arena_ralloc_large(void *ptr, size_t size, size_t oldsize)
    4709                 : {
    4710                 :         size_t psize;
    4711                 : 
    4712           28338 :         psize = PAGE_CEILING(size);
    4713           28338 :         if (psize == oldsize) {
    4714                 :                 /* Same size class. */
    4715                 : #ifdef MALLOC_FILL
    4716             108 :                 if (opt_junk && size < oldsize) {
    4717             108 :                         memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize -
    4718                 :                             size);
    4719                 :                 }
    4720                 : #endif
    4721             108 :                 return (false);
    4722                 :         } else {
    4723                 :                 arena_chunk_t *chunk;
    4724                 :                 arena_t *arena;
    4725                 : 
    4726           28230 :                 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
    4727           28230 :                 arena = chunk->arena;
    4728           28230 :                 assert(arena->magic == ARENA_MAGIC);
    4729                 : 
    4730           28230 :                 if (psize < oldsize) {
    4731                 : #ifdef MALLOC_FILL
    4732                 :                         /* Fill before shrinking in order avoid a race. */
    4733             117 :                         if (opt_junk) {
    4734             117 :                                 memset((void *)((uintptr_t)ptr + size), 0x5a,
    4735                 :                                     oldsize - size);
    4736                 :                         }
    4737                 : #endif
    4738             117 :                         arena_ralloc_large_shrink(arena, chunk, ptr, psize,
    4739                 :                             oldsize);
    4740             117 :                         return (false);
    4741                 :                 } else {
    4742           28113 :                         bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
    4743                 :                             psize, oldsize);
    4744                 : #ifdef MALLOC_FILL
    4745           28113 :                         if (ret == false && opt_zero) {
    4746               0 :                                 memset((void *)((uintptr_t)ptr + oldsize), 0,
    4747                 :                                     size - oldsize);
    4748                 :                         }
    4749                 : #endif
    4750           28113 :                         return (ret);
    4751                 :                 }
    4752                 :         }
    4753                 : }
    4754                 : 
    4755                 : static void *
    4756         2299975 : arena_ralloc(void *ptr, size_t size, size_t oldsize)
    4757                 : {
    4758                 :         void *ret;
    4759                 :         size_t copysize;
    4760                 : 
    4761                 :         /* Try to avoid moving the allocation. */
    4762         2299975 :         if (size < small_min) {
    4763          337330 :                 if (oldsize < small_min &&
    4764           83165 :                     ffs((int)(pow2_ceil(size) >> (TINY_MIN_2POW + 1)))
    4765           83165 :                     == ffs((int)(pow2_ceil(oldsize) >> (TINY_MIN_2POW + 1))))
    4766            9324 :                         goto IN_PLACE; /* Same size class. */
    4767         2045810 :         } else if (size <= small_max) {
    4768         3339456 :                 if (oldsize >= small_min && oldsize <= small_max &&
    4769         1653543 :                     (QUANTUM_CEILING(size) >> opt_quantum_2pow)
    4770         1653543 :                     == (QUANTUM_CEILING(oldsize) >> opt_quantum_2pow))
    4771           18432 :                         goto IN_PLACE; /* Same size class. */
    4772          359897 :         } else if (size <= bin_maxclass) {
    4773          393045 :                 if (oldsize > small_max && oldsize <= bin_maxclass &&
    4774           89419 :                     pow2_ceil(size) == pow2_ceil(oldsize))
    4775            2007 :                         goto IN_PLACE; /* Same size class. */
    4776           56271 :         } else if (oldsize > bin_maxclass && oldsize <= arena_maxclass) {
    4777           28338 :                 assert(size > bin_maxclass);
    4778           28338 :                 if (arena_ralloc_large(ptr, size, oldsize) == false)
    4779            9343 :                         return (ptr);
    4780                 :         }
    4781                 : 
    4782                 :         /*
    4783                 :          * If we get here, then size and oldsize are different enough that we
    4784                 :          * need to move the object.  In that case, fall back to allocating new
    4785                 :          * space and copying.
    4786                 :          */
    4787         2260869 :         ret = arena_malloc(choose_arena(), size, false);
    4788         2260869 :         if (ret == NULL)
    4789               0 :                 return (NULL);
    4790                 : 
    4791                 :         /* Junk/zero-filling were already done by arena_malloc(). */
    4792         2260869 :         copysize = (size < oldsize) ? size : oldsize;
    4793                 : #ifdef VM_COPY_MIN
    4794                 :         if (copysize >= VM_COPY_MIN)
    4795                 :                 pages_copy(ret, ptr, copysize);
    4796                 :         else
    4797                 : #endif
    4798         2260869 :                 memcpy(ret, ptr, copysize);
    4799         2260869 :         idalloc(ptr);
    4800         2260869 :         return (ret);
    4801                 : IN_PLACE:
    4802                 : #ifdef MALLOC_FILL
    4803           29763 :         if (opt_junk && size < oldsize)
    4804           20583 :                 memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size);
    4805            9180 :         else if (opt_zero && size > oldsize)
    4806               0 :                 memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize);
    4807                 : #endif
    4808           29763 :         return (ptr);
    4809                 : }
    4810                 : 
    4811                 : static inline void *
    4812         2301663 : iralloc(void *ptr, size_t size)
    4813                 : {
    4814                 :         size_t oldsize;
    4815                 : 
    4816         2301663 :         assert(ptr != NULL);
    4817         2301663 :         assert(size != 0);
    4818                 : 
    4819         2301663 :         oldsize = isalloc(ptr);
    4820                 : 
    4821                 : #ifndef MALLOC_VALGRIND
    4822         2301663 :         if (size <= arena_maxclass)
    4823         2299975 :                 return (arena_ralloc(ptr, size, oldsize));
    4824                 :         else
    4825            1688 :                 return (huge_ralloc(ptr, size, oldsize));
    4826                 : #else
    4827                 :         /*
    4828                 :          * Valgrind does not provide a public interface for modifying an
    4829                 :          * existing allocation, so use malloc/memcpy/free instead.
    4830                 :          */
    4831                 :         {
    4832                 :                 void *ret = imalloc(size);
    4833                 :                 if (ret != NULL) {
    4834                 :                         if (oldsize < size)
    4835                 :                             memcpy(ret, ptr, oldsize);
    4836                 :                         else
    4837                 :                             memcpy(ret, ptr, size);
    4838                 :                         idalloc(ptr);
    4839                 :                 }
    4840                 :                 return (ret);
    4841                 :         }
    4842                 : #endif
    4843                 : }
    4844                 : 
    4845                 : static bool
    4846           18669 : arena_new(arena_t *arena)
    4847                 : {
    4848                 :         unsigned i;
    4849                 :         arena_bin_t *bin;
    4850                 :         size_t pow2_size, prev_run_size;
    4851                 : 
    4852           18669 :         if (malloc_spin_init(&arena->lock))
    4853               0 :                 return (true);
    4854                 : 
    4855                 : #ifdef MALLOC_STATS
    4856           18669 :         memset(&arena->stats, 0, sizeof(arena_stats_t));
    4857                 : #endif
    4858                 : 
    4859                 :         /* Initialize chunks. */
    4860           18669 :         arena_chunk_tree_dirty_new(&arena->chunks_dirty);
    4861                 : #ifdef MALLOC_DOUBLE_PURGE
    4862                 :         LinkedList_Init(&arena->chunks_madvised);
    4863                 : #endif
    4864           18669 :         arena->spare = NULL;
    4865                 : 
    4866           18669 :         arena->ndirty = 0;
    4867                 : 
    4868           18669 :         arena_avail_tree_new(&arena->runs_avail);
    4869                 : 
    4870                 : #ifdef MALLOC_BALANCE
    4871                 :         arena->contention = 0;
    4872                 : #endif
    4873                 : 
    4874                 :         /* Initialize bins. */
    4875           18669 :         prev_run_size = pagesize;
    4876                 : 
    4877                 :         /* (2^n)-spaced tiny bins. */
    4878           56007 :         for (i = 0; i < ntbins; i++) {
    4879           37338 :                 bin = &arena->bins[i];
    4880           37338 :                 bin->runcur = NULL;
    4881           37338 :                 arena_run_tree_new(&bin->runs);
    4882                 : 
    4883           37338 :                 bin->reg_size = (1U << (TINY_MIN_2POW + i));
    4884                 : 
    4885           37338 :                 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
    4886                 : 
    4887                 : #ifdef MALLOC_STATS
    4888           37338 :                 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
    4889                 : #endif
    4890                 :         }
    4891                 : 
    4892                 :         /* Quantum-spaced bins. */
    4893          616077 :         for (; i < ntbins + nqbins; i++) {
    4894          597408 :                 bin = &arena->bins[i];
    4895          597408 :                 bin->runcur = NULL;
    4896          597408 :                 arena_run_tree_new(&bin->runs);
    4897                 : 
    4898          597408 :                 bin->reg_size = quantum * (i - ntbins + 1);
    4899                 : 
    4900          597408 :                 pow2_size = pow2_ceil(quantum * (i - ntbins + 1));
    4901          597408 :                 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
    4902                 : 
    4903                 : #ifdef MALLOC_STATS
    4904          597408 :                 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
    4905                 : #endif
    4906                 :         }
    4907                 : 
    4908                 :         /* (2^n)-spaced sub-page bins. */
    4909           56007 :         for (; i < ntbins + nqbins + nsbins; i++) {
    4910           37338 :                 bin = &arena->bins[i];
    4911           37338 :                 bin->runcur = NULL;
    4912           37338 :                 arena_run_tree_new(&bin->runs);
    4913                 : 
    4914           37338 :                 bin->reg_size = (small_max << (i - (ntbins + nqbins) + 1));
    4915                 : 
    4916           37338 :                 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
    4917                 : 
    4918                 : #ifdef MALLOC_STATS
    4919           37338 :                 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
    4920                 : #endif
    4921                 :         }
    4922                 : 
    4923                 : #ifdef MALLOC_DEBUG
    4924           18669 :         arena->magic = ARENA_MAGIC;
    4925                 : #endif
    4926                 : 
    4927           18669 :         return (false);
    4928                 : }
    4929                 : 
    4930                 : /* Create a new arena and insert it into the arenas array at index ind. */
    4931                 : static arena_t *
    4932           18669 : arenas_extend(unsigned ind)
    4933                 : {
    4934                 :         arena_t *ret;
    4935                 : 
    4936                 :         /* Allocate enough space for trailing bins. */
    4937           18669 :         ret = (arena_t *)base_alloc(sizeof(arena_t)
    4938           18669 :             + (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1)));
    4939           18669 :         if (ret != NULL && arena_new(ret) == false) {
    4940           18669 :                 arenas[ind] = ret;
    4941           18669 :                 return (ret);
    4942                 :         }
    4943                 :         /* Only reached if there is an OOM error. */
    4944                 : 
    4945                 :         /*
    4946                 :          * OOM here is quite inconvenient to propagate, since dealing with it
    4947                 :          * would require a check for failure in the fast path.  Instead, punt
    4948                 :          * by using arenas[0].  In practice, this is an extremely unlikely
    4949                 :          * failure.
    4950                 :          */
    4951               0 :         _malloc_message(_getprogname(),
    4952                 :             ": (malloc) Error initializing arena\n", "", "");
    4953               0 :         if (opt_abort)
    4954               0 :                 abort();
    4955                 : 
    4956               0 :         return (arenas[0]);
    4957                 : }
    4958                 : 
    4959                 : /*
    4960                 :  * End arena.
    4961                 :  */
    4962                 : /******************************************************************************/
    4963                 : /*
    4964                 :  * Begin general internal functions.
    4965                 :  */
    4966                 : 
    4967                 : static void *
    4968            4554 : huge_malloc(size_t size, bool zero)
    4969                 : {
    4970                 :         void *ret;
    4971                 :         size_t csize;
    4972                 :         size_t psize;
    4973                 :         extent_node_t *node;
    4974                 : 
    4975                 :         /* Allocate one or more contiguous chunks for this request. */
    4976                 : 
    4977            4554 :         csize = CHUNK_CEILING(size);
    4978            4554 :         if (csize == 0) {
    4979                 :                 /* size is large enough to cause size_t wrap-around. */
    4980               0 :                 return (NULL);
    4981                 :         }
    4982                 : 
    4983                 :         /* Allocate an extent node with which to track the chunk. */
    4984            4554 :         node = base_node_alloc();
    4985            4554 :         if (node == NULL)
    4986               0 :                 return (NULL);
    4987                 : 
    4988            4554 :         ret = chunk_alloc(csize, zero, true);
    4989            4554 :         if (ret == NULL) {
    4990              18 :                 base_node_dealloc(node);
    4991              18 :                 return (NULL);
    4992                 :         }
    4993                 : 
    4994                 :         /* Insert node into huge. */
    4995            4536 :         node->addr = ret;
    4996            4536 :         psize = PAGE_CEILING(size);
    4997            4536 :         node->size = psize;
    4998                 : 
    4999            4536 :         malloc_mutex_lock(&huge_mtx);
    5000            4536 :         extent_tree_ad_insert(&huge, node);
    5001                 : #ifdef MALLOC_STATS
    5002            4536 :         huge_nmalloc++;
    5003                 : 
    5004                 :         /* Although we allocated space for csize bytes, we indicate that we've
    5005                 :          * allocated only psize bytes.
    5006                 :          *
    5007                 :          * If DECOMMIT is defined, this is a reasonable thing to do, since
    5008                 :          * we'll explicitly decommit the bytes in excess of psize.
    5009                 :          *
    5010                 :          * If DECOMMIT is not defined, then we're relying on the OS to be lazy
    5011                 :          * about how it allocates physical pages to mappings.  If we never
    5012                 :          * touch the pages in excess of psize, the OS won't allocate a physical
    5013                 :          * page, and we won't use more than psize bytes of physical memory.
    5014                 :          *
    5015                 :          * A correct program will only touch memory in excess of how much it
    5016                 :          * requested if it first calls malloc_usable_size and finds out how
    5017                 :          * much space it has to play with.  But because we set node->size =
    5018                 :          * psize above, malloc_usable_size will return psize, not csize, and
    5019                 :          * the program will (hopefully) never touch bytes in excess of psize.
    5020                 :          * Thus those bytes won't take up space in physical memory, and we can
    5021                 :          * reasonably claim we never "allocated" them in the first place. */
    5022            4536 :         huge_allocated += psize;
    5023                 : #endif
    5024            4536 :         malloc_mutex_unlock(&huge_mtx);
    5025                 : 
    5026                 : #ifdef MALLOC_DECOMMIT
    5027                 :         if (csize - psize > 0)
    5028                 :                 pages_decommit((void *)((uintptr_t)ret + psize), csize - psize);
    5029                 : #endif
    5030                 : 
    5031                 : #ifdef MALLOC_DECOMMIT
    5032                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, zero);
    5033                 : #else
    5034                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, csize, 0, zero);
    5035                 : #endif
    5036                 : 
    5037                 : #ifdef MALLOC_FILL
    5038            4536 :         if (zero == false) {
    5039            4518 :                 if (opt_junk)
    5040                 : #  ifdef MALLOC_DECOMMIT
    5041                 :                         memset(ret, 0xa5, psize);
    5042                 : #  else
    5043            4518 :                         memset(ret, 0xa5, csize);
    5044                 : #  endif
    5045               0 :                 else if (opt_zero)
    5046                 : #  ifdef MALLOC_DECOMMIT
    5047                 :                         memset(ret, 0, psize);
    5048                 : #  else
    5049               0 :                         memset(ret, 0, csize);
    5050                 : #  endif
    5051                 :         }
    5052                 : #endif
    5053                 : 
    5054            4536 :         return (ret);
    5055                 : }
    5056                 : 
    5057                 : /* Only handles large allocations that require more than chunk alignment. */
    5058                 : static void *
    5059               0 : huge_palloc(size_t alignment, size_t size)
    5060                 : {
    5061                 :         void *ret;
    5062                 :         size_t alloc_size, chunk_size, offset;
    5063                 :         size_t psize;
    5064                 :         extent_node_t *node;
    5065                 :         int pfd;
    5066                 : 
    5067                 :         /*
    5068                 :          * This allocation requires alignment that is even larger than chunk
    5069                 :          * alignment.  This means that huge_malloc() isn't good enough.
    5070                 :          *
    5071                 :          * Allocate almost twice as many chunks as are demanded by the size or
    5072                 :          * alignment, in order to assure the alignment can be achieved, then
    5073                 :          * unmap leading and trailing chunks.
    5074                 :          */
    5075               0 :         assert(alignment >= chunksize);
    5076                 : 
    5077               0 :         chunk_size = CHUNK_CEILING(size);
    5078                 : 
    5079               0 :         if (size >= alignment)
    5080               0 :                 alloc_size = chunk_size + alignment - chunksize;
    5081                 :         else
    5082               0 :                 alloc_size = (alignment << 1) - chunksize;
    5083                 : 
    5084                 :         /* Allocate an extent node with which to track the chunk. */
    5085               0 :         node = base_node_alloc();
    5086               0 :         if (node == NULL)
    5087               0 :                 return (NULL);
    5088                 : 
    5089                 :         /*
    5090                 :          * Windows requires that there be a 1:1 mapping between VM
    5091                 :          * allocation/deallocation operations.  Therefore, take care here to
    5092                 :          * acquire the final result via one mapping operation.
    5093                 :          *
    5094                 :          * The MALLOC_PAGEFILE code also benefits from this mapping algorithm,
    5095                 :          * since it reduces the number of page files.
    5096                 :          */
    5097                 : #ifdef MALLOC_PAGEFILE
    5098                 :         if (opt_pagefile) {
    5099                 :                 pfd = pagefile_init(size);
    5100                 :                 if (pfd == -1)
    5101                 :                         return (NULL);
    5102                 :         } else
    5103                 : #endif
    5104               0 :                 pfd = -1;
    5105                 : #ifdef JEMALLOC_USES_MAP_ALIGN
    5106                 :                 ret = pages_map_align(chunk_size, pfd, alignment);
    5107                 : #else
    5108                 :         do {
    5109                 :                 void *over;
    5110                 : 
    5111               0 :                 over = chunk_alloc(alloc_size, false, false);
    5112               0 :                 if (over == NULL) {
    5113               0 :                         base_node_dealloc(node);
    5114               0 :                         ret = NULL;
    5115               0 :                         goto RETURN;
    5116                 :                 }
    5117                 : 
    5118               0 :                 offset = (uintptr_t)over & (alignment - 1);
    5119               0 :                 assert((offset & chunksize_mask) == 0);
    5120               0 :                 assert(offset < alloc_size);
    5121               0 :                 ret = (void *)((uintptr_t)over + offset);
    5122               0 :                 chunk_dealloc(over, alloc_size);
    5123               0 :                 ret = pages_map(ret, chunk_size, pfd);
    5124                 :                 /*
    5125                 :                  * Failure here indicates a race with another thread, so try
    5126                 :                  * again.
    5127                 :                  */
    5128               0 :         } while (ret == NULL);
    5129                 : #endif
    5130                 :         /* Insert node into huge. */
    5131               0 :         node->addr = ret;
    5132               0 :         psize = PAGE_CEILING(size);
    5133               0 :         node->size = psize;
    5134                 : 
    5135               0 :         malloc_mutex_lock(&huge_mtx);
    5136               0 :         extent_tree_ad_insert(&huge, node);
    5137                 : #ifdef MALLOC_STATS
    5138               0 :         huge_nmalloc++;
    5139                 :         /* See note in huge_alloc() for why huge_allocated += psize is correct
    5140                 :          * here even when DECOMMIT is not defined. */
    5141               0 :         huge_allocated += psize;
    5142                 : #endif
    5143               0 :         malloc_mutex_unlock(&huge_mtx);
    5144                 : 
    5145                 : #ifdef MALLOC_DECOMMIT
    5146                 :         if (chunk_size - psize > 0) {
    5147                 :                 pages_decommit((void *)((uintptr_t)ret + psize),
    5148                 :                     chunk_size - psize);
    5149                 :         }
    5150                 : #endif
    5151                 : 
    5152                 : #ifdef MALLOC_DECOMMIT
    5153                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, false);
    5154                 : #else
    5155                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, chunk_size, 0, false);
    5156                 : #endif
    5157                 : 
    5158                 : #ifdef MALLOC_FILL
    5159               0 :         if (opt_junk)
    5160                 : #  ifdef MALLOC_DECOMMIT
    5161                 :                 memset(ret, 0xa5, psize);
    5162                 : #  else
    5163               0 :                 memset(ret, 0xa5, chunk_size);
    5164                 : #  endif
    5165               0 :         else if (opt_zero)
    5166                 : #  ifdef MALLOC_DECOMMIT
    5167                 :                 memset(ret, 0, psize);
    5168                 : #  else
    5169               0 :                 memset(ret, 0, chunk_size);
    5170                 : #  endif
    5171                 : #endif
    5172                 : 
    5173                 : RETURN:
    5174                 : #ifdef MALLOC_PAGEFILE
    5175                 :         if (pfd != -1)
    5176                 :                 pagefile_close(pfd);
    5177                 : #endif
    5178               0 :         return (ret);
    5179                 : }
    5180                 : 
    5181                 : static void *
    5182            1688 : huge_ralloc(void *ptr, size_t size, size_t oldsize)
    5183                 : {
    5184                 :         void *ret;
    5185                 :         size_t copysize;
    5186                 : 
    5187                 :         /* Avoid moving the allocation if the size class would not change. */
    5188                 : 
    5189            3191 :         if (oldsize > arena_maxclass &&
    5190            1503 :             CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
    5191               0 :                 size_t psize = PAGE_CEILING(size);
    5192                 : #ifdef MALLOC_FILL
    5193               0 :                 if (opt_junk && size < oldsize) {
    5194               0 :                         memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize
    5195                 :                             - size);
    5196                 :                 }
    5197                 : #endif
    5198                 : #ifdef MALLOC_DECOMMIT
    5199                 :                 if (psize < oldsize) {
    5200                 :                         extent_node_t *node, key;
    5201                 : 
    5202                 :                         pages_decommit((void *)((uintptr_t)ptr + psize),
    5203                 :                             oldsize - psize);
    5204                 : 
    5205                 :                         /* Update recorded size. */
    5206                 :                         malloc_mutex_lock(&huge_mtx);
    5207                 :                         key.addr = __DECONST(void *, ptr);
    5208                 :                         node = extent_tree_ad_search(&huge, &key);
    5209                 :                         assert(node != NULL);
    5210                 :                         assert(node->size == oldsize);
    5211                 : #  ifdef MALLOC_STATS
    5212                 :                         huge_allocated -= oldsize - psize;
    5213                 : #  endif
    5214                 :                         node->size = psize;
    5215                 :                         malloc_mutex_unlock(&huge_mtx);
    5216                 :                 } else if (psize > oldsize) {
    5217                 :                         pages_commit((void *)((uintptr_t)ptr + oldsize),
    5218                 :                             psize - oldsize);
    5219                 :                 }
    5220                 : #endif
    5221                 : 
    5222                 :                 /* Although we don't have to commit or decommit anything if
    5223                 :                  * DECOMMIT is not defined and the size class didn't change, we
    5224                 :                  * do need to update the recorded size if the size increased,
    5225                 :                  * so malloc_usable_size doesn't return a value smaller than
    5226                 :                  * what was requested via realloc(). */
    5227                 : 
    5228               0 :                 if (psize > oldsize) {
    5229                 :                         /* Update recorded size. */
    5230                 :                         extent_node_t *node, key;
    5231               0 :                         malloc_mutex_lock(&huge_mtx);
    5232               0 :                         key.addr = __DECONST(void *, ptr);
    5233               0 :                         node = extent_tree_ad_search(&huge, &key);
    5234               0 :                         assert(node != NULL);
    5235               0 :                         assert(node->size == oldsize);
    5236                 : #  ifdef MALLOC_STATS
    5237               0 :                         huge_allocated += psize - oldsize;
    5238                 : #  endif
    5239               0 :                         node->size = psize;
    5240               0 :                         malloc_mutex_unlock(&huge_mtx);
    5241                 :                 }
    5242                 : 
    5243                 : #ifdef MALLOC_FILL
    5244               0 :                 if (opt_zero && size > oldsize) {
    5245               0 :                         memset((void *)((uintptr_t)ptr + oldsize), 0, size
    5246                 :                             - oldsize);
    5247                 :                 }
    5248                 : #endif
    5249               0 :                 return (ptr);
    5250                 :         }
    5251                 : 
    5252                 :         /*
    5253                 :          * If we get here, then size and oldsize are different enough that we
    5254                 :          * need to use a different size class.  In that case, fall back to
    5255                 :          * allocating new space and copying.
    5256                 :          */
    5257            1688 :         ret = huge_malloc(size, false);
    5258            1688 :         if (ret == NULL)
    5259               0 :                 return (NULL);
    5260                 : 
    5261            1688 :         copysize = (size < oldsize) ? size : oldsize;
    5262                 : #ifdef VM_COPY_MIN
    5263                 :         if (copysize >= VM_COPY_MIN)
    5264                 :                 pages_copy(ret, ptr, copysize);
    5265                 :         else
    5266                 : #endif
    5267            1688 :                 memcpy(ret, ptr, copysize);
    5268            1688 :         idalloc(ptr);
    5269            1688 :         return (ret);
    5270                 : }
    5271                 : 
    5272                 : static void
    5273            4536 : huge_dalloc(void *ptr)
    5274                 : {
    5275                 :         extent_node_t *node, key;
    5276                 : 
    5277            4536 :         malloc_mutex_lock(&huge_mtx);
    5278                 : 
    5279                 :         /* Extract from tree of huge allocations. */
    5280            4536 :         key.addr = ptr;
    5281            4536 :         node = extent_tree_ad_search(&huge, &key);
    5282            4536 :         assert(node != NULL);
    5283            4536 :         assert(node->addr == ptr);
    5284            4536 :         extent_tree_ad_remove(&huge, node);
    5285                 : 
    5286                 : #ifdef MALLOC_STATS
    5287            4536 :         huge_ndalloc++;
    5288            4536 :         huge_allocated -= node->size;
    5289                 : #endif
    5290                 : 
    5291            4536 :         malloc_mutex_unlock(&huge_mtx);
    5292                 : 
    5293                 :         /* Unmap chunk. */
    5294                 : #ifdef MALLOC_FILL
    5295            4536 :         if (opt_junk)
    5296            4536 :                 memset(node->addr, 0x5a, node->size);
    5297                 : #endif
    5298            4536 :         chunk_dealloc(node->addr, CHUNK_CEILING(node->size));
    5299                 :         VALGRIND_FREELIKE_BLOCK(node->addr, 0);
    5300                 : 
    5301            4536 :         base_node_dealloc(node);
    5302            4536 : }
    5303                 : 
    5304                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    5305                 : #ifdef MOZ_MEMORY_BSD
    5306                 : static inline unsigned
    5307                 : malloc_ncpus(void)
    5308                 : {
    5309                 :         unsigned ret;
    5310                 :         int mib[2];
    5311                 :         size_t len;
    5312                 : 
    5313                 :         mib[0] = CTL_HW;
    5314                 :         mib[1] = HW_NCPU;
    5315                 :         len = sizeof(ret);
    5316                 :         if (sysctl(mib, 2, &ret, &len, (void *) 0, 0) == -1) {
    5317                 :                 /* Error. */
    5318                 :                 return (1);
    5319                 :         }
    5320                 : 
    5321                 :         return (ret);
    5322                 : }
    5323                 : #elif (defined(MOZ_MEMORY_LINUX))
    5324                 : #include <fcntl.h>
    5325                 : 
    5326                 : static inline unsigned
    5327                 : malloc_ncpus(void)
    5328                 : {
    5329                 :         unsigned ret;
    5330                 :         int fd, nread, column;
    5331                 :         char buf[1024];
    5332                 :         static const char matchstr[] = "processor\t:";
    5333                 :         int i;
    5334                 : 
    5335                 :         /*
    5336                 :          * sysconf(3) would be the preferred method for determining the number
    5337                 :          * of CPUs, but it uses malloc internally, which causes untennable
    5338                 :          * recursion during malloc initialization.
    5339                 :          */
    5340                 :         fd = open("/proc/cpuinfo", O_RDONLY);
    5341                 :         if (fd == -1)
    5342                 :                 return (1); /* Error. */
    5343                 :         /*
    5344                 :          * Count the number of occurrences of matchstr at the beginnings of
    5345                 :          * lines.  This treats hyperthreaded CPUs as multiple processors.
    5346                 :          */
    5347                 :         column = 0;
    5348                 :         ret = 0;
    5349                 :         while (true) {
    5350                 :                 nread = read(fd, &buf, sizeof(buf));
    5351                 :                 if (nread <= 0)
    5352                 :                         break; /* EOF or error. */
    5353                 :                 for (i = 0;i < nread;i++) {
    5354                 :                         char c = buf[i];
    5355                 :                         if (c == '\n')
    5356                 :                                 column = 0;
    5357                 :                         else if (column != -1) {
    5358                 :                                 if (c == matchstr[column]) {
    5359                 :                                         column++;
    5360                 :                                         if (column == sizeof(matchstr) - 1) {
    5361                 :                                                 column = -1;
    5362                 :                                                 ret++;
    5363                 :                                         }
    5364                 :                                 } else
    5365                 :                                         column = -1;
    5366                 :                         }
    5367                 :                 }
    5368                 :         }
    5369                 : 
    5370                 :         if (ret == 0)
    5371                 :                 ret = 1; /* Something went wrong in the parser. */
    5372                 :         close(fd);
    5373                 : 
    5374                 :         return (ret);
    5375                 : }
    5376                 : #elif (defined(MOZ_MEMORY_DARWIN))
    5377                 : #include <mach/mach_init.h>
    5378                 : #include <mach/mach_host.h>
    5379                 : 
    5380                 : static inline unsigned
    5381                 : malloc_ncpus(void)
    5382                 : {
    5383                 :         kern_return_t error;
    5384                 :         natural_t n;
    5385                 :         processor_info_array_t pinfo;
    5386                 :         mach_msg_type_number_t pinfocnt;
    5387                 : 
    5388                 :         error = host_processor_info(mach_host_self(), PROCESSOR_BASIC_INFO,
    5389                 :                                     &n, &pinfo, &pinfocnt);
    5390                 :         if (error != KERN_SUCCESS)
    5391                 :                 return (1); /* Error. */
    5392                 :         else
    5393                 :                 return (n);
    5394                 : }
    5395                 : #elif (defined(MOZ_MEMORY_SOLARIS))
    5396                 : 
    5397                 : static inline unsigned
    5398                 : malloc_ncpus(void)
    5399                 : {
    5400                 :         return sysconf(_SC_NPROCESSORS_ONLN);
    5401                 : }
    5402                 : #else
    5403                 : static inline unsigned
    5404                 : malloc_ncpus(void)
    5405                 : {
    5406                 : 
    5407                 :         /*
    5408                 :          * We lack a way to determine the number of CPUs on this platform, so
    5409                 :          * assume 1 CPU.
    5410                 :          */
    5411                 :         return (1);
    5412                 : }
    5413                 : #endif
    5414                 : #endif
    5415                 : 
    5416                 : static void
    5417               0 : malloc_print_stats(void)
    5418                 : {
    5419                 : 
    5420               0 :         if (opt_print_stats) {
    5421                 :                 char s[UMAX2S_BUFSIZE];
    5422               0 :                 _malloc_message("___ Begin malloc statistics ___\n", "", "",
    5423                 :                     "");
    5424               0 :                 _malloc_message("Assertions ",
    5425                 : #ifdef NDEBUG
    5426                 :                     "disabled",
    5427                 : #else
    5428                 :                     "enabled",
    5429                 : #endif
    5430                 :                     "\n", "");
    5431               0 :                 _malloc_message("Boolean MALLOC_OPTIONS: ",
    5432                 :                     opt_abort ? "A" : "a", "", "");
    5433                 : #ifdef MALLOC_FILL
    5434               0 :                 _malloc_message(opt_junk ? "J" : "j", "", "", "");
    5435                 : #endif
    5436                 : #ifdef MALLOC_PAGEFILE
    5437                 :                 _malloc_message(opt_pagefile ? "o" : "O", "", "", "");
    5438                 : #endif
    5439               0 :                 _malloc_message("P", "", "", "");
    5440                 : #ifdef MALLOC_UTRACE
    5441               0 :                 _malloc_message(opt_utrace ? "U" : "u", "", "", "");
    5442                 : #endif
    5443                 : #ifdef MALLOC_SYSV
    5444               0 :                 _malloc_message(opt_sysv ? "V" : "v", "", "", "");
    5445                 : #endif
    5446                 : #ifdef MALLOC_XMALLOC
    5447               0 :                 _malloc_message(opt_xmalloc ? "X" : "x", "", "", "");
    5448                 : #endif
    5449                 : #ifdef MALLOC_FILL
    5450               0 :                 _malloc_message(opt_zero ? "Z" : "z", "", "", "");
    5451                 : #endif
    5452               0 :                 _malloc_message("\n", "", "", "");
    5453                 : 
    5454                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    5455                 :                 _malloc_message("CPUs: ", umax2s(ncpus, 10, s), "\n", "");
    5456                 : #endif
    5457               0 :                 _malloc_message("Max arenas: ", umax2s(narenas, 10, s), "\n",
    5458                 :                     "");
    5459                 : #ifdef MALLOC_BALANCE
    5460                 :                 _malloc_message("Arena balance threshold: ",
    5461                 :                     umax2s(opt_balance_threshold, 10, s), "\n", "");
    5462                 : #endif
    5463               0 :                 _malloc_message("Pointer size: ", umax2s(sizeof(void *), 10, s),
    5464                 :                     "\n", "");
    5465               0 :                 _malloc_message("Quantum size: ", umax2s(quantum, 10, s), "\n",
    5466                 :                     "");
    5467               0 :                 _malloc_message("Max small size: ", umax2s(small_max, 10, s),
    5468                 :                     "\n", "");
    5469               0 :                 _malloc_message("Max dirty pages per arena: ",
    5470               0 :                     umax2s(opt_dirty_max, 10, s), "\n", "");
    5471                 : 
    5472               0 :                 _malloc_message("Chunk size: ", umax2s(chunksize, 10, s), "",
    5473                 :                     "");
    5474               0 :                 _malloc_message(" (2^", umax2s(opt_chunk_2pow, 10, s), ")\n",
    5475                 :                     "");
    5476                 : 
    5477                 : #ifdef MALLOC_STATS
    5478                 :                 {
    5479                 :                         size_t allocated, mapped;
    5480                 : #ifdef MALLOC_BALANCE
    5481                 :                         uint64_t nbalance = 0;
    5482                 : #endif
    5483                 :                         unsigned i;
    5484                 :                         arena_t *arena;
    5485                 : 
    5486                 :                         /* Calculate and print allocated/mapped stats. */
    5487                 : 
    5488                 :                         /* arenas. */
    5489               0 :                         for (i = 0, allocated = 0; i < narenas; i++) {
    5490               0 :                                 if (arenas[i] != NULL) {
    5491               0 :                                         malloc_spin_lock(&arenas[i]->lock);
    5492               0 :                                         allocated +=
    5493               0 :                                             arenas[i]->stats.allocated_small;
    5494               0 :                                         allocated +=
    5495               0 :                                             arenas[i]->stats.allocated_large;
    5496                 : #ifdef MALLOC_BALANCE
    5497                 :                                         nbalance += arenas[i]->stats.nbalance;
    5498                 : #endif
    5499               0 :                                         malloc_spin_unlock(&arenas[i]->lock);
    5500                 :                                 }
    5501                 :                         }
    5502                 : 
    5503                 :                         /* huge/base. */
    5504               0 :                         malloc_mutex_lock(&huge_mtx);
    5505               0 :                         allocated += huge_allocated;
    5506               0 :                         mapped = stats_chunks.curchunks * chunksize;
    5507               0 :                         malloc_mutex_unlock(&huge_mtx);
    5508                 : 
    5509               0 :                         malloc_mutex_lock(&base_mtx);
    5510               0 :                         mapped += base_mapped;
    5511               0 :                         malloc_mutex_unlock(&base_mtx);
    5512                 : 
    5513                 : #ifdef MOZ_MEMORY_WINDOWS
    5514                 :                         malloc_printf("Allocated: %lu, mapped: %lu\n",
    5515                 :                             allocated, mapped);
    5516                 : #else
    5517               0 :                         malloc_printf("Allocated: %zu, mapped: %zu\n",
    5518                 :                             allocated, mapped);
    5519                 : #endif
    5520                 : 
    5521                 : #ifdef MALLOC_BALANCE
    5522                 :                         malloc_printf("Arena balance reassignments: %llu\n",
    5523                 :                             nbalance);
    5524                 : #endif
    5525                 : 
    5526                 :                         /* Print chunk stats. */
    5527                 :                         {
    5528                 :                                 chunk_stats_t chunks_stats;
    5529                 : 
    5530               0 :                                 malloc_mutex_lock(&huge_mtx);
    5531               0 :                                 chunks_stats = stats_chunks;
    5532               0 :                                 malloc_mutex_unlock(&huge_mtx);
    5533                 : 
    5534               0 :                                 malloc_printf("chunks: nchunks   "
    5535                 :                                     "highchunks    curchunks\n");
    5536               0 :                                 malloc_printf("  %13llu%13lu%13lu\n",
    5537                 :                                     chunks_stats.nchunks,
    5538                 :                                     chunks_stats.highchunks,
    5539                 :                                     chunks_stats.curchunks);
    5540                 :                         }
    5541                 : 
    5542                 :                         /* Print chunk stats. */
    5543               0 :                         malloc_printf(
    5544                 :                             "huge: nmalloc      ndalloc    allocated\n");
    5545                 : #ifdef MOZ_MEMORY_WINDOWS
    5546                 :                         malloc_printf(" %12llu %12llu %12lu\n",
    5547                 :                             huge_nmalloc, huge_ndalloc, huge_allocated);
    5548                 : #else
    5549               0 :                         malloc_printf(" %12llu %12llu %12zu\n",
    5550                 :                             huge_nmalloc, huge_ndalloc, huge_allocated);
    5551                 : #endif
    5552                 :                         /* Print stats for each arena. */
    5553               0 :                         for (i = 0; i < narenas; i++) {
    5554               0 :                                 arena = arenas[i];
    5555               0 :                                 if (arena != NULL) {
    5556               0 :                                         malloc_printf(
    5557                 :                                             "\narenas[%u]:\n", i);
    5558               0 :                                         malloc_spin_lock(&arena->lock);
    5559               0 :                                         stats_print(arena);
    5560               0 :                                         malloc_spin_unlock(&arena->lock);
    5561                 :                                 }
    5562                 :                         }
    5563                 :                 }
    5564                 : #endif /* #ifdef MALLOC_STATS */
    5565               0 :                 _malloc_message("--- End malloc statistics ---\n", "", "", "");
    5566                 :         }
    5567               0 : }
    5568                 : 
    5569                 : /*
    5570                 :  * FreeBSD's pthreads implementation calls malloc(3), so the malloc
    5571                 :  * implementation has to take pains to avoid infinite recursion during
    5572                 :  * initialization.
    5573                 :  */
    5574                 : #if (defined(MOZ_MEMORY_WINDOWS) || defined(MOZ_MEMORY_DARWIN))
    5575                 : #define malloc_init() false
    5576                 : #else
    5577                 : static inline bool
    5578        44114699 : malloc_init(void)
    5579                 : {
    5580                 : 
    5581        44114699 :         if (malloc_initialized == false)
    5582           18669 :                 return (malloc_init_hard());
    5583                 : 
    5584        44096030 :         return (false);
    5585                 : }
    5586                 : #endif
    5587                 : 
    5588                 : #if !defined(MOZ_MEMORY_WINDOWS)
    5589                 : static
    5590                 : #endif
    5591                 : bool
    5592           18669 : malloc_init_hard(void)
    5593                 : {
    5594                 :         unsigned i;
    5595                 :         char buf[PATH_MAX + 1];
    5596                 :         const char *opts;
    5597                 :         long result;
    5598                 : #ifndef MOZ_MEMORY_WINDOWS
    5599                 :         int linklen;
    5600                 : #endif
    5601                 : #ifdef MOZ_MEMORY_DARWIN
    5602                 :     malloc_zone_t* default_zone;
    5603                 : #endif
    5604                 : 
    5605                 : #ifndef MOZ_MEMORY_WINDOWS
    5606           18669 :         malloc_mutex_lock(&init_lock);
    5607                 : #endif
    5608                 : 
    5609           18669 :         if (malloc_initialized) {
    5610                 :                 /*
    5611                 :                  * Another thread initialized the allocator before this one
    5612                 :                  * acquired init_lock.
    5613                 :                  */
    5614                 : #ifndef MOZ_MEMORY_WINDOWS
    5615               0 :                 malloc_mutex_unlock(&init_lock);
    5616                 : #endif
    5617               0 :                 return (false);
    5618                 :         }
    5619                 : 
    5620                 : #ifdef MOZ_MEMORY_WINDOWS
    5621                 :         /* get a thread local storage index */
    5622                 :         tlsIndex = TlsAlloc();
    5623                 : #endif
    5624                 : 
    5625                 :         /* Get page size and number of CPUs */
    5626                 : #ifdef MOZ_MEMORY_WINDOWS
    5627                 :         {
    5628                 :                 SYSTEM_INFO info;
    5629                 : 
    5630                 :                 GetSystemInfo(&info);
    5631                 :                 result = info.dwPageSize;
    5632                 : 
    5633                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    5634                 :                 ncpus = info.dwNumberOfProcessors;
    5635                 : #endif
    5636                 :         }
    5637                 : #else
    5638                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    5639                 :         ncpus = malloc_ncpus();
    5640                 : #endif
    5641                 : 
    5642           18669 :         result = sysconf(_SC_PAGESIZE);
    5643           18669 :         assert(result != -1);
    5644                 : #endif
    5645                 : 
    5646                 :         /* We assume that the page size is a power of 2. */
    5647           18669 :         assert(((result - 1) & result) == 0);
    5648                 : #ifdef MALLOC_STATIC_SIZES
    5649           18669 :         if (pagesize % (size_t) result) {
    5650               0 :                 _malloc_message(_getprogname(),
    5651                 :                                 "Compile-time page size does not divide the runtime one.\n",
    5652                 :                                 "", "");
    5653               0 :                 abort();
    5654                 :         }
    5655                 : #else   
    5656                 :         pagesize = (size_t) result;
    5657                 :         pagesize_mask = (size_t) result - 1;
    5658                 :         pagesize_2pow = ffs((int)result) - 1;
    5659                 : #endif
    5660                 :         
    5661                 : #ifdef MALLOC_PAGEFILE
    5662                 :         /*
    5663                 :          * Determine where to create page files.  It is insufficient to
    5664                 :          * unconditionally use P_tmpdir (typically "/tmp"), since for some
    5665                 :          * operating systems /tmp is a separate filesystem that is rather small.
    5666                 :          * Therefore prefer, in order, the following locations:
    5667                 :          *
    5668                 :          * 1) MALLOC_TMPDIR
    5669                 :          * 2) TMPDIR
    5670                 :          * 3) P_tmpdir
    5671                 :          */
    5672                 :         {
    5673                 :                 char *s;
    5674                 :                 size_t slen;
    5675                 :                 static const char suffix[] = "/jemalloc.XXXXXX";
    5676                 : 
    5677                 :                 if ((s = getenv("MALLOC_TMPDIR")) == NULL && (s =
    5678                 :                     getenv("TMPDIR")) == NULL)
    5679                 :                         s = P_tmpdir;
    5680                 :                 slen = strlen(s);
    5681                 :                 if (slen + sizeof(suffix) > sizeof(pagefile_templ)) {
    5682                 :                         _malloc_message(_getprogname(),
    5683                 :                             ": (malloc) Page file path too long\n",
    5684                 :                             "", "");
    5685                 :                         abort();
    5686                 :                 }
    5687                 :                 memcpy(pagefile_templ, s, slen);
    5688                 :                 memcpy(&pagefile_templ[slen], suffix, sizeof(suffix));
    5689                 :         }
    5690                 : #endif
    5691                 : 
    5692           74676 :         for (i = 0; i < 3; i++) {
    5693                 :                 unsigned j;
    5694                 : 
    5695                 :                 /* Get runtime configuration. */
    5696           56007 :                 switch (i) {
    5697                 :                 case 0:
    5698                 : #ifndef MOZ_MEMORY_WINDOWS
    5699           18669 :                         if ((linklen = readlink("/etc/malloc.conf", buf,
    5700                 :                                                 sizeof(buf) - 1)) != -1) {
    5701                 :                                 /*
    5702                 :                                  * Use the contents of the "/etc/malloc.conf"
    5703                 :                                  * symbolic link's name.
    5704                 :                                  */
    5705               0 :                                 buf[linklen] = '\0';
    5706               0 :                                 opts = buf;
    5707                 :                         } else
    5708                 : #endif
    5709                 :                         {
    5710                 :                                 /* No configuration specified. */
    5711           18669 :                                 buf[0] = '\0';
    5712           18669 :                                 opts = buf;
    5713                 :                         }
    5714           18669 :                         break;
    5715                 :                 case 1:
    5716           18669 :                         if (issetugid() == 0 && (opts =
    5717                 :                             getenv("MALLOC_OPTIONS")) != NULL) {
    5718                 :                                 /*
    5719                 :                                  * Do nothing; opts is already initialized to
    5720                 :                                  * the value of the MALLOC_OPTIONS environment
    5721                 :                                  * variable.
    5722                 :                                  */
    5723                 :                         } else {
    5724                 :                                 /* No configuration specified. */
    5725           18669 :                                 buf[0] = '\0';
    5726           18669 :                                 opts = buf;
    5727                 :                         }
    5728           18669 :                         break;
    5729                 :                 case 2:
    5730           18669 :                         if (_malloc_options != NULL) {
    5731                 :                                 /*
    5732                 :                                  * Use options that were compiled into the
    5733                 :                                  * program.
    5734                 :                                  */
    5735               0 :                                 opts = _malloc_options;
    5736                 :                         } else {
    5737                 :                                 /* No configuration specified. */
    5738           18669 :                                 buf[0] = '\0';
    5739           18669 :                                 opts = buf;
    5740                 :                         }
    5741           18669 :                         break;
    5742                 :                 default:
    5743                 :                         /* NOTREACHED */
    5744               0 :                         buf[0] = '\0';
    5745               0 :                         opts = buf;
    5746               0 :                         assert(false);
    5747                 :                 }
    5748                 : 
    5749           56007 :                 for (j = 0; opts[j] != '\0'; j++) {
    5750                 :                         unsigned k, nreps;
    5751                 :                         bool nseen;
    5752                 : 
    5753                 :                         /* Parse repetition count, if any. */
    5754               0 :                         for (nreps = 0, nseen = false;; j++, nseen = true) {
    5755               0 :                                 switch (opts[j]) {
    5756                 :                                         case '0': case '1': case '2': case '3':
    5757                 :                                         case '4': case '5': case '6': case '7':
    5758                 :                                         case '8': case '9':
    5759               0 :                                                 nreps *= 10;
    5760               0 :                                                 nreps += opts[j] - '0';
    5761                 :                                                 break;
    5762                 :                                         default:
    5763               0 :                                                 goto MALLOC_OUT;
    5764                 :                                 }
    5765               0 :                         }
    5766                 : MALLOC_OUT:
    5767               0 :                         if (nseen == false)
    5768               0 :                                 nreps = 1;
    5769                 : 
    5770               0 :                         for (k = 0; k < nreps; k++) {
    5771               0 :                                 switch (opts[j]) {
    5772                 :                                 case 'a':
    5773               0 :                                         opt_abort = false;
    5774               0 :                                         break;
    5775                 :                                 case 'A':
    5776               0 :                                         opt_abort = true;
    5777               0 :                                         break;
    5778                 :                                 case 'b':
    5779                 : #ifdef MALLOC_BALANCE
    5780                 :                                         opt_balance_threshold >>= 1;
    5781                 : #endif
    5782               0 :                                         break;
    5783                 :                                 case 'B':
    5784                 : #ifdef MALLOC_BALANCE
    5785                 :                                         if (opt_balance_threshold == 0)
    5786                 :                                                 opt_balance_threshold = 1;
    5787                 :                                         else if ((opt_balance_threshold << 1)
    5788                 :                                             > opt_balance_threshold)
    5789                 :                                                 opt_balance_threshold <<= 1;
    5790                 : #endif
    5791               0 :                                         break;
    5792                 :                                 case 'f':
    5793               0 :                                         opt_dirty_max >>= 1;
    5794               0 :                                         break;
    5795                 :                                 case 'F':
    5796               0 :                                         if (opt_dirty_max == 0)
    5797               0 :                                                 opt_dirty_max = 1;
    5798               0 :                                         else if ((opt_dirty_max << 1) != 0)
    5799               0 :                                                 opt_dirty_max <<= 1;
    5800               0 :                                         break;
    5801                 : #ifdef MALLOC_FILL
    5802                 :                                 case 'j':
    5803               0 :                                         opt_junk = false;
    5804               0 :                                         break;
    5805                 :                                 case 'J':
    5806               0 :                                         opt_junk = true;
    5807               0 :                                         break;
    5808                 : #endif
    5809                 : #ifndef MALLOC_STATIC_SIZES
    5810                 :                                 case 'k':
    5811                 :                                         /*
    5812                 :                                          * Chunks always require at least one
    5813                 :                                          * header page, so chunks can never be
    5814                 :                                          * smaller than two pages.
    5815                 :                                          */
    5816                 :                                         if (opt_chunk_2pow > pagesize_2pow + 1)
    5817                 :                                                 opt_chunk_2pow--;
    5818                 :                                         break;
    5819                 :                                 case 'K':
    5820                 :                                         if (opt_chunk_2pow + 1 <
    5821                 :                                             (sizeof(size_t) << 3))
    5822                 :                                                 opt_chunk_2pow++;
    5823                 :                                         break;
    5824                 : #endif
    5825                 :                                 case 'n':
    5826               0 :                                         opt_narenas_lshift--;
    5827               0 :                                         break;
    5828                 :                                 case 'N':
    5829               0 :                                         opt_narenas_lshift++;
    5830               0 :                                         break;
    5831                 : #ifdef MALLOC_PAGEFILE
    5832                 :                                 case 'o':
    5833                 :                                         /* Do not over-commit. */
    5834                 :                                         opt_pagefile = true;
    5835                 :                                         break;
    5836                 :                                 case 'O':
    5837                 :                                         /* Allow over-commit. */
    5838                 :                                         opt_pagefile = false;
    5839                 :                                         break;
    5840                 : #endif
    5841                 :                                 case 'p':
    5842               0 :                                         opt_print_stats = false;
    5843               0 :                                         break;
    5844                 :                                 case 'P':
    5845               0 :                                         opt_print_stats = true;
    5846               0 :                                         break;
    5847                 : #ifndef MALLOC_STATIC_SIZES
    5848                 :                                 case 'q':
    5849                 :                                         if (opt_quantum_2pow > QUANTUM_2POW_MIN)
    5850                 :                                                 opt_quantum_2pow--;
    5851                 :                                         break;
    5852                 :                                 case 'Q':
    5853                 :                                         if (opt_quantum_2pow < pagesize_2pow -
    5854                 :                                             1)
    5855                 :                                                 opt_quantum_2pow++;
    5856                 :                                         break;
    5857                 :                                 case 's':
    5858                 :                                         if (opt_small_max_2pow >
    5859                 :                                             QUANTUM_2POW_MIN)
    5860                 :                                                 opt_small_max_2pow--;
    5861                 :                                         break;
    5862                 :                                 case 'S':
    5863                 :                                         if (opt_small_max_2pow < pagesize_2pow
    5864                 :                                             - 1)
    5865                 :                                                 opt_small_max_2pow++;
    5866                 :                                         break;
    5867                 : #endif
    5868                 : #ifdef MALLOC_UTRACE
    5869                 :                                 case 'u':
    5870               0 :                                         opt_utrace = false;
    5871               0 :                                         break;
    5872                 :                                 case 'U':
    5873               0 :                                         opt_utrace = true;
    5874               0 :                                         break;
    5875                 : #endif
    5876                 : #ifdef MALLOC_SYSV
    5877                 :                                 case 'v':
    5878               0 :                                         opt_sysv = false;
    5879               0 :                                         break;
    5880                 :                                 case 'V':
    5881               0 :                                         opt_sysv = true;
    5882               0 :                                         break;
    5883                 : #endif
    5884                 : #ifdef MALLOC_XMALLOC
    5885                 :                                 case 'x':
    5886               0 :                                         opt_xmalloc = false;
    5887               0 :                                         break;
    5888                 :                                 case 'X':
    5889               0 :                                         opt_xmalloc = true;
    5890               0 :                                         break;
    5891                 : #endif
    5892                 : #ifdef MALLOC_FILL
    5893                 :                                 case 'z':
    5894               0 :                                         opt_zero = false;
    5895               0 :                                         break;
    5896                 :                                 case 'Z':
    5897               0 :                                         opt_zero = true;
    5898               0 :                                         break;
    5899                 : #endif
    5900                 :                                 default: {
    5901                 :                                         char cbuf[2];
    5902                 : 
    5903               0 :                                         cbuf[0] = opts[j];
    5904               0 :                                         cbuf[1] = '\0';
    5905               0 :                                         _malloc_message(_getprogname(),
    5906                 :                                             ": (malloc) Unsupported character "
    5907                 :                                             "in malloc options: '", cbuf,
    5908                 :                                             "'\n");
    5909                 :                                 }
    5910                 :                                 }
    5911                 :                         }
    5912                 :                 }
    5913                 :         }
    5914                 : 
    5915                 :         /* Take care to call atexit() only once. */
    5916           18669 :         if (opt_print_stats) {
    5917                 : #ifndef MOZ_MEMORY_WINDOWS
    5918                 :                 /* Print statistics at exit. */
    5919               0 :                 atexit(malloc_print_stats);
    5920                 : #endif
    5921                 :         }
    5922                 : 
    5923                 : #if !defined(MOZ_MEMORY_WINDOWS) && !defined(MOZ_MEMORY_DARWIN)
    5924                 :         /* Prevent potential deadlock on malloc locks after fork. */
    5925           18669 :         pthread_atfork(_malloc_prefork, _malloc_postfork, _malloc_postfork);
    5926                 : #endif
    5927                 : 
    5928                 : #ifndef MALLOC_STATIC_SIZES
    5929                 :         /* Set variables according to the value of opt_small_max_2pow. */
    5930                 :         if (opt_small_max_2pow < opt_quantum_2pow)
    5931                 :                 opt_small_max_2pow = opt_quantum_2pow;
    5932                 :         small_max = (1U << opt_small_max_2pow);
    5933                 : 
    5934                 :         /* Set bin-related variables. */
    5935                 :         bin_maxclass = (pagesize >> 1);
    5936                 :         assert(opt_quantum_2pow >= TINY_MIN_2POW);
    5937                 :         ntbins = opt_quantum_2pow - TINY_MIN_2POW;
    5938                 :         assert(ntbins <= opt_quantum_2pow);
    5939                 :         nqbins = (small_max >> opt_quantum_2pow);
    5940                 :         nsbins = pagesize_2pow - opt_small_max_2pow - 1;
    5941                 : 
    5942                 :         /* Set variables according to the value of opt_quantum_2pow. */
    5943                 :         quantum = (1U << opt_quantum_2pow);
    5944                 :         quantum_mask = quantum - 1;
    5945                 :         if (ntbins > 0)
    5946                 :                 small_min = (quantum >> 1) + 1;
    5947                 :         else
    5948                 :                 small_min = 1;
    5949                 :         assert(small_min <= quantum);
    5950                 : 
    5951                 :         /* Set variables according to the value of opt_chunk_2pow. */
    5952                 :         chunksize = (1LU << opt_chunk_2pow);
    5953                 :         chunksize_mask = chunksize - 1;
    5954                 :         chunk_npages = (chunksize >> pagesize_2pow);
    5955                 : 
    5956                 :         arena_chunk_header_npages = calculate_arena_header_pages();
    5957                 :         arena_maxclass = calculate_arena_maxclass();
    5958                 : #endif
    5959                 : 
    5960                 : #ifdef JEMALLOC_USES_MAP_ALIGN
    5961                 :         /*
    5962                 :          * When using MAP_ALIGN, the alignment parameter must be a power of two
    5963                 :          * multiple of the system pagesize, or mmap will fail.
    5964                 :          */
    5965                 :         assert((chunksize % pagesize) == 0);
    5966                 :         assert((1 << (ffs(chunksize / pagesize) - 1)) == (chunksize/pagesize));
    5967                 : #endif
    5968                 : 
    5969           18669 :         UTRACE(0, 0, 0);
    5970                 : 
    5971                 : #ifdef MALLOC_STATS
    5972           18669 :         memset(&stats_chunks, 0, sizeof(chunk_stats_t));
    5973                 : #endif
    5974                 : 
    5975                 :         /* Various sanity checks that regard configuration. */
    5976           18669 :         assert(quantum >= sizeof(void *));
    5977           18669 :         assert(quantum <= pagesize);
    5978           18669 :         assert(chunksize >= pagesize);
    5979           18669 :         assert(quantum * 4 <= chunksize);
    5980                 : 
    5981                 :         /* Initialize chunks data. */
    5982           18669 :         malloc_mutex_init(&huge_mtx);
    5983           18669 :         extent_tree_ad_new(&huge);
    5984                 : #ifdef MALLOC_STATS
    5985           18669 :         huge_nmalloc = 0;
    5986           18669 :         huge_ndalloc = 0;
    5987           18669 :         huge_allocated = 0;
    5988                 : #endif
    5989                 : 
    5990                 :         /* Initialize base allocation data structures. */
    5991                 : #ifdef MALLOC_STATS
    5992           18669 :         base_mapped = 0;
    5993           18669 :         base_committed = 0;
    5994                 : #endif
    5995           18669 :         base_nodes = NULL;
    5996           18669 :         malloc_mutex_init(&base_mtx);
    5997                 : 
    5998                 : #ifdef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    5999           18669 :         narenas = 1;
    6000                 : #else
    6001                 :         if (ncpus > 1) {
    6002                 :                 /*
    6003                 :                  * For SMP systems, create four times as many arenas as there
    6004                 :                  * are CPUs by default.
    6005                 :                  */
    6006                 :                 opt_narenas_lshift += 2;
    6007                 :         }
    6008                 : 
    6009                 :         /* Determine how many arenas to use. */
    6010                 :         narenas = ncpus;
    6011                 : #endif
    6012           18669 :         if (opt_narenas_lshift > 0) {
    6013               0 :                 if ((narenas << opt_narenas_lshift) > narenas)
    6014               0 :                         narenas <<= opt_narenas_lshift;
    6015                 :                 /*
    6016                 :                  * Make sure not to exceed the limits of what base_alloc() can
    6017                 :                  * handle.
    6018                 :                  */
    6019               0 :                 if (narenas * sizeof(arena_t *) > chunksize)
    6020               0 :                         narenas = chunksize / sizeof(arena_t *);
    6021           18669 :         } else if (opt_narenas_lshift < 0) {
    6022               0 :                 if ((narenas >> -opt_narenas_lshift) < narenas)
    6023               0 :                         narenas >>= -opt_narenas_lshift;
    6024                 :                 /* Make sure there is at least one arena. */
    6025               0 :                 if (narenas == 0)
    6026               0 :                         narenas = 1;
    6027                 :         }
    6028                 : #ifdef MALLOC_BALANCE
    6029                 :         assert(narenas != 0);
    6030                 :         for (narenas_2pow = 0;
    6031                 :              (narenas >> (narenas_2pow + 1)) != 0;
    6032                 :              narenas_2pow++);
    6033                 : #endif
    6034                 : 
    6035                 : #ifdef NO_TLS
    6036                 :         if (narenas > 1) {
    6037                 :                 static const unsigned primes[] = {1, 3, 5, 7, 11, 13, 17, 19,
    6038                 :                     23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,
    6039                 :                     89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149,
    6040                 :                     151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211,
    6041                 :                     223, 227, 229, 233, 239, 241, 251, 257, 263};
    6042                 :                 unsigned nprimes, parenas;
    6043                 : 
    6044                 :                 /*
    6045                 :                  * Pick a prime number of hash arenas that is more than narenas
    6046                 :                  * so that direct hashing of pthread_self() pointers tends to
    6047                 :                  * spread allocations evenly among the arenas.
    6048                 :                  */
    6049                 :                 assert((narenas & 1) == 0); /* narenas must be even. */
    6050                 :                 nprimes = (sizeof(primes) >> SIZEOF_INT_2POW);
    6051                 :                 parenas = primes[nprimes - 1]; /* In case not enough primes. */
    6052                 :                 for (i = 1; i < nprimes; i++) {
    6053                 :                         if (primes[i] > narenas) {
    6054                 :                                 parenas = primes[i];
    6055                 :                                 break;
    6056                 :                         }
    6057                 :                 }
    6058                 :                 narenas = parenas;
    6059                 :         }
    6060                 : #endif
    6061                 : 
    6062                 : #ifndef NO_TLS
    6063                 : #  ifndef MALLOC_BALANCE
    6064           18669 :         next_arena = 0;
    6065                 : #  endif
    6066                 : #endif
    6067                 : 
    6068                 :         /* Allocate and initialize arenas. */
    6069           18669 :         arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
    6070           18669 :         if (arenas == NULL) {
    6071                 : #ifndef MOZ_MEMORY_WINDOWS
    6072               0 :                 malloc_mutex_unlock(&init_lock);
    6073                 : #endif
    6074               0 :                 return (true);
    6075                 :         }
    6076                 :         /*
    6077                 :          * Zero the array.  In practice, this should always be pre-zeroed,
    6078                 :          * since it was just mmap()ed, but let's be sure.
    6079                 :          */
    6080           18669 :         memset(arenas, 0, sizeof(arena_t *) * narenas);
    6081                 : 
    6082                 :         /*
    6083                 :          * Initialize one arena here.  The rest are lazily created in
    6084                 :          * choose_arena_hard().
    6085                 :          */
    6086           18669 :         arenas_extend(0);
    6087           18669 :         if (arenas[0] == NULL) {
    6088                 : #ifndef MOZ_MEMORY_WINDOWS
    6089               0 :                 malloc_mutex_unlock(&init_lock);
    6090                 : #endif
    6091               0 :                 return (true);
    6092                 :         }
    6093                 : #ifndef NO_TLS
    6094                 :         /*
    6095                 :          * Assign the initial arena to the initial thread, in order to avoid
    6096                 :          * spurious creation of an extra arena if the application switches to
    6097                 :          * threaded mode.
    6098                 :          */
    6099                 : #ifdef MOZ_MEMORY_WINDOWS
    6100                 :         TlsSetValue(tlsIndex, arenas[0]);
    6101                 : #else
    6102           18669 :         arenas_map = arenas[0];
    6103                 : #endif
    6104                 : #endif
    6105                 : 
    6106                 :         /*
    6107                 :          * Seed here for the initial thread, since choose_arena_hard() is only
    6108                 :          * called for other threads.  The seed value doesn't really matter.
    6109                 :          */
    6110                 : #ifdef MALLOC_BALANCE
    6111                 :         SPRN(balance, 42);
    6112                 : #endif
    6113                 : 
    6114           18669 :         malloc_spin_init(&arenas_lock);
    6115                 : 
    6116                 : #ifdef MALLOC_VALIDATE
    6117           18669 :         chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - opt_chunk_2pow);
    6118           18669 :         if (chunk_rtree == NULL)
    6119               0 :                 return (true);
    6120                 : #endif
    6121                 : 
    6122           18669 :         malloc_initialized = true;
    6123                 : 
    6124                 : #if defined(NEEDS_PTHREAD_MMAP_UNALIGNED_TSD)
    6125                 :         if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
    6126                 :                 malloc_printf("<jemalloc>: Error in pthread_key_create()\n");
    6127                 :         }
    6128                 : #endif
    6129                 : 
    6130                 : #ifdef MOZ_MEMORY_DARWIN
    6131                 :         /*
    6132                 :         * Overwrite the default memory allocator to use jemalloc everywhere.
    6133                 :         */
    6134                 :         default_zone = malloc_default_zone();
    6135                 : 
    6136                 :         /*
    6137                 :          * We only use jemalloc with MacOS 10.6 and 10.7.  jemalloc is disabled
    6138                 :          * on 32-bit builds (10.5 and 32-bit 10.6) due to bug 702250, an
    6139                 :          * apparent MacOS bug.  In fact, this code isn't even compiled on
    6140                 :          * 32-bit builds.
    6141                 :          *
    6142                 :          * We'll have to update our code to work with newer versions, because
    6143                 :          * the malloc zone layout is likely to change.
    6144                 :          */
    6145                 : 
    6146                 :         osx_use_jemalloc = (default_zone->version == SNOW_LEOPARD_MALLOC_ZONE_T_VERSION ||
    6147                 :                             default_zone->version == LION_MALLOC_ZONE_T_VERSION);
    6148                 : 
    6149                 :         /* Allow us dynamically turn off jemalloc for testing. */
    6150                 :         if (getenv("NO_MAC_JEMALLOC")) {
    6151                 :                 osx_use_jemalloc = false;
    6152                 : #ifdef __i386__
    6153                 :                 malloc_printf("Warning: NO_MAC_JEMALLOC has no effect on "
    6154                 :                               "i386 machines (such as this one).\n");
    6155                 : #endif
    6156                 :         }
    6157                 : 
    6158                 :         if (osx_use_jemalloc) {
    6159                 :                 /*
    6160                 :                  * Convert the default szone to an "overlay zone" that is capable
    6161                 :                  * of deallocating szone-allocated objects, but allocating new
    6162                 :                  * objects from jemalloc.
    6163                 :                  */
    6164                 :                 size_t size = zone_version_size(default_zone->version);
    6165                 :                 szone2ozone(default_zone, size);
    6166                 :         }
    6167                 :         else {
    6168                 :                 szone = default_zone;
    6169                 :         }
    6170                 : #endif
    6171                 : 
    6172                 : #ifndef MOZ_MEMORY_WINDOWS
    6173           18669 :         malloc_mutex_unlock(&init_lock);
    6174                 : #endif
    6175           18669 :         return (false);
    6176                 : }
    6177                 : 
    6178                 : /* XXX Why not just expose malloc_print_stats()? */
    6179                 : #ifdef MOZ_MEMORY_WINDOWS
    6180                 : void
    6181                 : malloc_shutdown()
    6182                 : {
    6183                 : 
    6184                 :         malloc_print_stats();
    6185                 : }
    6186                 : #endif
    6187                 : 
    6188                 : /*
    6189                 :  * End general internal functions.
    6190                 :  */
    6191                 : /******************************************************************************/
    6192                 : /*
    6193                 :  * Begin malloc(3)-compatible functions.
    6194                 :  */
    6195                 : 
    6196                 : /*
    6197                 :  * Mangle standard interfaces, in order to avoid linking problems.
    6198                 :  */
    6199                 : #if defined(MOZ_MEMORY_DARWIN) || defined(MOZ_MEMORY_WINDOWS) || \
    6200                 :     defined(MOZ_MEMORY_ANDROID)
    6201                 : 
    6202                 : #ifdef MOZ_MEMORY_ANDROID
    6203                 : /*
    6204                 :  * On Android, we use __wrap_* instead of je_* to accomodate with the
    6205                 :  * linker's --wrap option we use. That option prefixes the function
    6206                 :  * names it is given with __wrap_.
    6207                 :  */
    6208                 : #define wrap(a) __wrap_ ## a
    6209                 : #else
    6210                 : #define wrap(a) je_ ## a
    6211                 : #endif
    6212                 : 
    6213                 : #define malloc(a)               wrap(malloc)(a)
    6214                 : #define memalign(a, b)          wrap(memalign)(a, b)
    6215                 : #define posix_memalign(a, b, c) wrap(posix_memalign)(a, b, c)
    6216                 : #define valloc(a)               wrap(valloc)(a)
    6217                 : #define calloc(a, b)            wrap(calloc)(a, b)
    6218                 : #define realloc(a, b)           wrap(realloc)(a, b)
    6219                 : #define free(a)                 wrap(free)(a)
    6220                 : #define malloc_usable_size(a)   wrap(malloc_usable_size)(a)
    6221                 : #endif
    6222                 : 
    6223                 : /*
    6224                 :  * Even though we compile with MOZ_MEMORY, we may have to dynamically decide
    6225                 :  * not to use jemalloc, as discussed above. However, we call jemalloc
    6226                 :  * functions directly from mozalloc. Since it's pretty dangerous to mix the
    6227                 :  * allocators, we need to call the OSX allocators from the functions below,
    6228                 :  * when osx_use_jemalloc is not (dynamically) set.
    6229                 :  *
    6230                 :  * Note that we assume jemalloc is enabled on i386.  This is safe because the
    6231                 :  * only i386 versions of MacOS are 10.5 and 10.6, which we support.  We have to
    6232                 :  * do this because madvise isn't in the malloc zone struct for 10.5.
    6233                 :  *
    6234                 :  * This means that NO_MAC_JEMALLOC doesn't work on i386.
    6235                 :  */
    6236                 : #if defined(MOZ_MEMORY_DARWIN) && !defined(__i386__)
    6237                 : #define DARWIN_ONLY(A) if (!osx_use_jemalloc) { A; }
    6238                 : #else
    6239                 : #define DARWIN_ONLY(A)
    6240                 : #endif
    6241                 : 
    6242                 : void *
    6243                 : malloc(size_t size)
    6244                 : {
    6245                 :         void *ret;
    6246                 : 
    6247                 :         DARWIN_ONLY(return (szone->malloc)(szone, size));
    6248                 : 
    6249        37094556 :         if (malloc_init()) {
    6250               0 :                 ret = NULL;
    6251               0 :                 goto RETURN;
    6252                 :         }
    6253                 : 
    6254        37094556 :         if (size == 0) {
    6255                 : #ifdef MALLOC_SYSV
    6256              22 :                 if (opt_sysv == false)
    6257                 : #endif
    6258              22 :                         size = 1;
    6259                 : #ifdef MALLOC_SYSV
    6260                 :                 else {
    6261               0 :                         ret = NULL;
    6262               0 :                         goto RETURN;
    6263                 :                 }
    6264                 : #endif
    6265                 :         }
    6266                 : 
    6267        37094556 :         ret = imalloc(size);
    6268                 : 
    6269                 : RETURN:
    6270        37094556 :         if (ret == NULL) {
    6271                 : #ifdef MALLOC_XMALLOC
    6272               0 :                 if (opt_xmalloc) {
    6273               0 :                         _malloc_message(_getprogname(),
    6274                 :                             ": (malloc) Error in malloc(): out of memory\n", "",
    6275                 :                             "");
    6276               0 :                         abort();
    6277                 :                 }
    6278                 : #endif
    6279               0 :                 errno = ENOMEM;
    6280                 :         }
    6281                 : 
    6282        37094556 :         UTRACE(0, size, ret);
    6283        37094556 :         return (ret);
    6284                 : }
    6285                 : 
    6286                 : /*
    6287                 :  * In ELF systems the default visibility allows symbols to be preempted at
    6288                 :  * runtime. This in turn prevents the uses of memalign in this file from being
    6289                 :  * optimized. What we do in here is define two aliasing symbols (they point to
    6290                 :  * the same code): memalign and memalign_internal. The internal version has
    6291                 :  * hidden visibility and is used in every reference from this file.
    6292                 :  *
    6293                 :  * For more information on this technique, see section 2.2.7 (Avoid Using
    6294                 :  * Exported Symbols) in http://www.akkadia.org/drepper/dsohowto.pdf.
    6295                 :  */
    6296                 : 
    6297                 : #if defined(__GNUC__) && !defined(MOZ_MEMORY_DARWIN)
    6298                 : #define MOZ_MEMORY_ELF
    6299                 : #endif
    6300                 : 
    6301                 : #ifdef MOZ_MEMORY_SOLARIS
    6302                 : #  ifdef __SUNPRO_C
    6303                 : void *
    6304                 : memalign(size_t alignment, size_t size);
    6305                 : #pragma no_inline(memalign)
    6306                 : #  elif (defined(__GNUC__))
    6307                 : __attribute__((noinline))
    6308                 : #  endif
    6309                 : #else
    6310                 : #if (defined(MOZ_MEMORY_ELF))
    6311                 : __attribute__((visibility ("hidden")))
    6312                 : #endif
    6313                 : #endif
    6314                 : 
    6315                 : 
    6316                 : #ifdef MOZ_MEMORY_ELF
    6317                 : #define MEMALIGN memalign_internal
    6318                 : #else
    6319                 : #define MEMALIGN memalign
    6320                 : #endif
    6321                 : 
    6322                 : void *
    6323               0 : MEMALIGN(size_t alignment, size_t size)
    6324                 : {
    6325                 :         void *ret;
    6326                 : 
    6327                 :         DARWIN_ONLY(return (szone->memalign)(szone, alignment, size));
    6328                 : 
    6329               0 :         assert(((alignment - 1) & alignment) == 0);
    6330                 : 
    6331               0 :         if (malloc_init()) {
    6332               0 :                 ret = NULL;
    6333               0 :                 goto RETURN;
    6334                 :         }
    6335                 : 
    6336               0 :         if (size == 0) {
    6337                 : #ifdef MALLOC_SYSV
    6338               0 :                 if (opt_sysv == false)
    6339                 : #endif
    6340               0 :                         size = 1;
    6341                 : #ifdef MALLOC_SYSV
    6342                 :                 else {
    6343               0 :                         ret = NULL;
    6344               0 :                         goto RETURN;
    6345                 :                 }
    6346                 : #endif
    6347                 :         }
    6348                 : 
    6349               0 :         alignment = alignment < sizeof(void*) ? sizeof(void*) : alignment;
    6350               0 :         ret = ipalloc(alignment, size);
    6351                 : 
    6352                 : RETURN:
    6353                 : #ifdef MALLOC_XMALLOC
    6354               0 :         if (opt_xmalloc && ret == NULL) {
    6355               0 :                 _malloc_message(_getprogname(),
    6356                 :                 ": (malloc) Error in memalign(): out of memory\n", "", "");
    6357               0 :                 abort();
    6358                 :         }
    6359                 : #endif
    6360               0 :         UTRACE(0, size, ret);
    6361               0 :         return (ret);
    6362                 : }
    6363                 : 
    6364                 : #ifdef MOZ_MEMORY_ELF
    6365                 : extern void *
    6366                 : memalign(size_t alignment, size_t size) __attribute__((alias ("memalign_internal"), visibility ("default")));
    6367                 : #endif
    6368                 : 
    6369                 : int
    6370                 : posix_memalign(void **memptr, size_t alignment, size_t size)
    6371                 : {
    6372                 :         void *result;
    6373                 : 
    6374                 :         /* Make sure that alignment is a large enough power of 2. */
    6375               0 :         if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) {
    6376                 : #ifdef MALLOC_XMALLOC
    6377               0 :                 if (opt_xmalloc) {
    6378               0 :                         _malloc_message(_getprogname(),
    6379                 :                             ": (malloc) Error in posix_memalign(): "
    6380                 :                             "invalid alignment\n", "", "");
    6381               0 :                         abort();
    6382                 :                 }
    6383                 : #endif
    6384               0 :                 return (EINVAL);
    6385                 :         }
    6386                 : 
    6387                 :         /* The 0-->1 size promotion is done in the memalign() call below */
    6388                 : 
    6389               0 :         result = MEMALIGN(alignment, size);
    6390                 : 
    6391               0 :         if (result == NULL)
    6392               0 :                 return (ENOMEM);
    6393                 : 
    6394               0 :         *memptr = result;
    6395               0 :         return (0);
    6396                 : }
    6397                 : 
    6398                 : void *
    6399                 : valloc(size_t size)
    6400                 : {
    6401               0 :         return (MEMALIGN(pagesize, size));
    6402                 : }
    6403                 : 
    6404                 : void *
    6405                 : calloc(size_t num, size_t size)
    6406                 : {
    6407                 :         void *ret;
    6408                 :         size_t num_size;
    6409                 : 
    6410                 :         DARWIN_ONLY(return (szone->calloc)(szone, num, size));
    6411                 : 
    6412         2256419 :         if (malloc_init()) {
    6413               0 :                 num_size = 0;
    6414               0 :                 ret = NULL;
    6415               0 :                 goto RETURN;
    6416                 :         }
    6417                 : 
    6418         2256419 :         num_size = num * size;
    6419         2256419 :         if (num_size == 0) {
    6420                 : #ifdef MALLOC_SYSV
    6421               0 :                 if ((opt_sysv == false) && ((num == 0) || (size == 0)))
    6422                 : #endif
    6423               0 :                         num_size = 1;
    6424                 : #ifdef MALLOC_SYSV
    6425                 :                 else {
    6426               0 :                         ret = NULL;
    6427               0 :                         goto RETURN;
    6428                 :                 }
    6429                 : #endif
    6430                 :         /*
    6431                 :          * Try to avoid division here.  We know that it isn't possible to
    6432                 :          * overflow during multiplication if neither operand uses any of the
    6433                 :          * most significant half of the bits in a size_t.
    6434                 :          */
    6435         2256419 :         } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
    6436             551 :             && (num_size / size != num)) {
    6437                 :                 /* size_t overflow. */
    6438               0 :                 ret = NULL;
    6439               0 :                 goto RETURN;
    6440                 :         }
    6441                 : 
    6442         2256419 :         ret = icalloc(num_size);
    6443                 : 
    6444                 : RETURN:
    6445         2256419 :         if (ret == NULL) {
    6446                 : #ifdef MALLOC_XMALLOC
    6447              18 :                 if (opt_xmalloc) {
    6448               0 :                         _malloc_message(_getprogname(),
    6449                 :                             ": (malloc) Error in calloc(): out of memory\n", "",
    6450                 :                             "");
    6451               0 :                         abort();
    6452                 :                 }
    6453                 : #endif
    6454              18 :                 errno = ENOMEM;
    6455                 :         }
    6456                 : 
    6457         2256419 :         UTRACE(0, num_size, ret);
    6458         2256419 :         return (ret);
    6459                 : }
    6460                 : 
    6461                 : void *
    6462                 : realloc(void *ptr, size_t size)
    6463                 : {
    6464                 :         void *ret;
    6465                 : 
    6466                 :         DARWIN_ONLY(return (szone->realloc)(szone, ptr, size));
    6467                 : 
    6468         7065387 :         if (size == 0) {
    6469                 : #ifdef MALLOC_SYSV
    6470               0 :                 if (opt_sysv == false)
    6471                 : #endif
    6472               0 :                         size = 1;
    6473                 : #ifdef MALLOC_SYSV
    6474                 :                 else {
    6475               0 :                         if (ptr != NULL)
    6476               0 :                                 idalloc(ptr);
    6477               0 :                         ret = NULL;
    6478               0 :                         goto RETURN;
    6479                 :                 }
    6480                 : #endif
    6481                 :         }
    6482                 : 
    6483         7065387 :         if (ptr != NULL) {
    6484         2301663 :                 assert(malloc_initialized);
    6485                 : 
    6486         2301663 :                 ret = iralloc(ptr, size);
    6487                 : 
    6488         2301663 :                 if (ret == NULL) {
    6489                 : #ifdef MALLOC_XMALLOC
    6490               0 :                         if (opt_xmalloc) {
    6491               0 :                                 _malloc_message(_getprogname(),
    6492                 :                                     ": (malloc) Error in realloc(): out of "
    6493                 :                                     "memory\n", "", "");
    6494               0 :                                 abort();
    6495                 :                         }
    6496                 : #endif
    6497               0 :                         errno = ENOMEM;
    6498                 :                 }
    6499                 :         } else {
    6500         4763724 :                 if (malloc_init())
    6501               0 :                         ret = NULL;
    6502                 :                 else
    6503         4763724 :                         ret = imalloc(size);
    6504                 : 
    6505         4763724 :                 if (ret == NULL) {
    6506                 : #ifdef MALLOC_XMALLOC
    6507               0 :                         if (opt_xmalloc) {
    6508               0 :                                 _malloc_message(_getprogname(),
    6509                 :                                     ": (malloc) Error in realloc(): out of "
    6510                 :                                     "memory\n", "", "");
    6511               0 :                                 abort();
    6512                 :                         }
    6513                 : #endif
    6514               0 :                         errno = ENOMEM;
    6515                 :                 }
    6516                 :         }
    6517                 : 
    6518                 : #ifdef MALLOC_SYSV
    6519                 : RETURN:
    6520                 : #endif
    6521         7065387 :         UTRACE(ptr, size, ret);
    6522         7065387 :         return (ret);
    6523                 : }
    6524                 : 
    6525                 : void
    6526                 : free(void *ptr)
    6527                 : {
    6528                 :         size_t offset;
    6529                 :         
    6530                 :         DARWIN_ONLY((szone->free)(szone, ptr); return);
    6531                 : 
    6532        50933286 :         UTRACE(ptr, 0, 0);
    6533                 : 
    6534                 :         /*
    6535                 :          * A version of idalloc that checks for NULL pointer but only for
    6536                 :          * huge allocations assuming that CHUNK_ADDR2OFFSET(NULL) == 0.
    6537                 :          */
    6538                 :         assert(CHUNK_ADDR2OFFSET(NULL) == 0);
    6539        50933289 :         offset = CHUNK_ADDR2OFFSET(ptr);
    6540        50933289 :         if (offset != 0)
    6541        42036375 :                 arena_dalloc(ptr, offset);
    6542         8896914 :         else if (ptr != NULL)
    6543            2916 :                 huge_dalloc(ptr);
    6544        50933300 : }
    6545                 : 
    6546                 : /*
    6547                 :  * End malloc(3)-compatible functions.
    6548                 :  */
    6549                 : /******************************************************************************/
    6550                 : /*
    6551                 :  * Begin non-standard functions.
    6552                 :  */
    6553                 : 
    6554                 : /* This was added by Mozilla for use by SQLite. */
    6555                 : #ifdef MOZ_MEMORY_DARWIN
    6556                 : static
    6557                 : #endif
    6558                 : size_t
    6559               0 : je_malloc_good_size(size_t size)
    6560                 : {
    6561                 :         /*
    6562                 :          * This duplicates the logic in imalloc(), arena_malloc() and
    6563                 :          * arena_malloc_small().
    6564                 :          */
    6565               0 :         if (size < small_min) {
    6566                 :                 /* Small (tiny). */
    6567               0 :                 size = pow2_ceil(size);
    6568                 :                 /*
    6569                 :                  * We omit the #ifdefs from arena_malloc_small() --
    6570                 :                  * it can be inaccurate with its size in some cases, but this
    6571                 :                  * function must be accurate.
    6572                 :                  */
    6573               0 :                 if (size < (1U << TINY_MIN_2POW))
    6574               0 :                         size = (1U << TINY_MIN_2POW);
    6575               0 :         } else if (size <= small_max) {
    6576                 :                 /* Small (quantum-spaced). */
    6577               0 :                 size = QUANTUM_CEILING(size);
    6578               0 :         } else if (size <= bin_maxclass) {
    6579                 :                 /* Small (sub-page). */
    6580               0 :                 size = pow2_ceil(size);
    6581               0 :         } else if (size <= arena_maxclass) {
    6582                 :                 /* Large. */
    6583               0 :                 size = PAGE_CEILING(size);
    6584                 :         } else {
    6585                 :                 /*
    6586                 :                  * Huge.  We use PAGE_CEILING to get psize, instead of using
    6587                 :                  * CHUNK_CEILING to get csize.  This ensures that this
    6588                 :                  * malloc_usable_size(malloc(n)) always matches
    6589                 :                  * je_malloc_good_size(n).
    6590                 :                  */
    6591               0 :                 size = PAGE_CEILING(size);
    6592                 :         }
    6593               0 :         return size;
    6594                 : }
    6595                 : 
    6596                 : 
    6597                 : #ifdef MOZ_MEMORY_ANDROID
    6598                 : size_t
    6599                 : malloc_usable_size(void *ptr)
    6600                 : #else
    6601                 : size_t
    6602               0 : malloc_usable_size(const void *ptr)
    6603                 : #endif
    6604                 : {
    6605                 :         DARWIN_ONLY(return (szone->size)(szone, ptr));
    6606                 : 
    6607                 : #ifdef MALLOC_VALIDATE
    6608               0 :         return (isalloc_validate(ptr));
    6609                 : #else
    6610                 :         assert(ptr != NULL);
    6611                 : 
    6612                 :         return (isalloc(ptr));
    6613                 : #endif
    6614                 : }
    6615                 : 
    6616                 : void
    6617               0 : jemalloc_stats(jemalloc_stats_t *stats)
    6618                 : {
    6619                 :         size_t i;
    6620                 : 
    6621               0 :         assert(stats != NULL);
    6622                 : 
    6623                 :         /*
    6624                 :          * Gather runtime settings.
    6625                 :          */
    6626               0 :         stats->opt_abort = opt_abort;
    6627               0 :         stats->opt_junk =
    6628                 : #ifdef MALLOC_FILL
    6629                 :             opt_junk ? true :
    6630                 : #endif
    6631                 :             false;
    6632               0 :         stats->opt_utrace =
    6633                 : #ifdef MALLOC_UTRACE
    6634                 :             opt_utrace ? true :
    6635                 : #endif
    6636                 :             false;
    6637               0 :         stats->opt_sysv =
    6638                 : #ifdef MALLOC_SYSV
    6639                 :             opt_sysv ? true :
    6640                 : #endif
    6641                 :             false;
    6642               0 :         stats->opt_xmalloc =
    6643                 : #ifdef MALLOC_XMALLOC
    6644                 :             opt_xmalloc ? true :
    6645                 : #endif
    6646                 :             false;
    6647               0 :         stats->opt_zero =
    6648                 : #ifdef MALLOC_FILL
    6649                 :             opt_zero ? true :
    6650                 : #endif
    6651                 :             false;
    6652               0 :         stats->narenas = narenas;
    6653               0 :         stats->balance_threshold =
    6654                 : #ifdef MALLOC_BALANCE
    6655                 :             opt_balance_threshold
    6656                 : #else
    6657                 :             SIZE_T_MAX
    6658                 : #endif
    6659                 :             ;
    6660               0 :         stats->quantum = quantum;
    6661               0 :         stats->small_max = small_max;
    6662               0 :         stats->large_max = arena_maxclass;
    6663               0 :         stats->chunksize = chunksize;
    6664               0 :         stats->dirty_max = opt_dirty_max;
    6665                 : 
    6666                 :         /*
    6667                 :          * Gather current memory usage statistics.
    6668                 :          */
    6669               0 :         stats->mapped = 0;
    6670               0 :         stats->committed = 0;
    6671               0 :         stats->allocated = 0;
    6672               0 :         stats->dirty = 0;
    6673                 : 
    6674                 :         /* Get huge mapped/allocated. */
    6675               0 :         malloc_mutex_lock(&huge_mtx);
    6676               0 :         stats->mapped += stats_chunks.curchunks * chunksize;
    6677               0 :         stats->committed += huge_allocated;
    6678               0 :         stats->allocated += huge_allocated;
    6679               0 :         malloc_mutex_unlock(&huge_mtx);
    6680                 : 
    6681                 :         /* Get base mapped. */
    6682               0 :         malloc_mutex_lock(&base_mtx);
    6683               0 :         stats->mapped += base_mapped;
    6684               0 :         assert(base_committed <= base_mapped);
    6685               0 :         stats->committed += base_committed;
    6686               0 :         malloc_mutex_unlock(&base_mtx);
    6687                 : 
    6688                 :         /* Iterate over arenas and their chunks. */
    6689               0 :         for (i = 0; i < narenas; i++) {
    6690               0 :                 arena_t *arena = arenas[i];
    6691               0 :                 if (arena != NULL) {
    6692               0 :                         malloc_spin_lock(&arena->lock);
    6693               0 :                         stats->allocated += arena->stats.allocated_small;
    6694               0 :                         stats->allocated += arena->stats.allocated_large;
    6695               0 :                         stats->committed += (arena->stats.committed <<
    6696                 :                             pagesize_2pow);
    6697               0 :                         stats->dirty += (arena->ndirty << pagesize_2pow);
    6698               0 :                         malloc_spin_unlock(&arena->lock);
    6699                 :                 }
    6700                 :         }
    6701                 : 
    6702               0 :         assert(stats->mapped >= stats->committed);
    6703               0 :         assert(stats->committed >= stats->allocated);
    6704               0 : }
    6705                 : 
    6706                 : #ifdef MALLOC_DOUBLE_PURGE
    6707                 : 
    6708                 : /* Explicitly remove all of this chunk's MADV_FREE'd pages from memory. */
    6709                 : static void
    6710                 : hard_purge_chunk(arena_chunk_t *chunk)
    6711                 : {
    6712                 :         /* See similar logic in arena_purge(). */
    6713                 : 
    6714                 :         size_t i;
    6715                 :         for (i = arena_chunk_header_npages; i < chunk_npages; i++) {
    6716                 :                 /* Find all adjacent pages with CHUNK_MAP_MADVISED set. */
    6717                 :                 size_t npages;
    6718                 :                 for (npages = 0;
    6719                 :                      chunk->map[i + npages].bits & CHUNK_MAP_MADVISED && i + npages < chunk_npages;
    6720                 :                      npages++) {
    6721                 :                         /* Turn off the chunk's MADV_FREED bit and turn on its
    6722                 :                          * DECOMMITTED bit. */
    6723                 :                         assert(!(chunk->map[i + npages].bits & CHUNK_MAP_DECOMMITTED));
    6724                 :                         chunk->map[i + npages].bits ^= CHUNK_MAP_MADVISED_OR_DECOMMITTED;
    6725                 :                 }
    6726                 : 
    6727                 :                 /* We could use mincore to find out which pages are actually
    6728                 :                  * present, but it's not clear that's better. */
    6729                 :                 if (npages > 0) {
    6730                 :                         pages_decommit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
    6731                 :                         pages_commit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
    6732                 :                 }
    6733                 :                 i += npages;
    6734                 :         }
    6735                 : }
    6736                 : 
    6737                 : /* Explicitly remove all of this arena's MADV_FREE'd pages from memory. */
    6738                 : static void
    6739                 : hard_purge_arena(arena_t *arena)
    6740                 : {
    6741                 :         malloc_spin_lock(&arena->lock);
    6742                 : 
    6743                 :         while (!LinkedList_IsEmpty(&arena->chunks_madvised)) {
    6744                 :                 LinkedList* next = arena->chunks_madvised.next;
    6745                 :                 arena_chunk_t *chunk =
    6746                 :                         LinkedList_Get(arena->chunks_madvised.next,
    6747                 :                                        arena_chunk_t, chunks_madvised_elem);
    6748                 :                 hard_purge_chunk(chunk);
    6749                 :                 LinkedList_Remove(&chunk->chunks_madvised_elem);
    6750                 :         }
    6751                 : 
    6752                 :         malloc_spin_unlock(&arena->lock);
    6753                 : }
    6754                 : 
    6755                 : void
    6756                 : jemalloc_purge_freed_pages()
    6757                 : {
    6758                 :         size_t i;
    6759                 :         for (i = 0; i < narenas; i++) {
    6760                 :                 arena_t *arena = arenas[i];
    6761                 :                 if (arena != NULL)
    6762                 :                         hard_purge_arena(arena);
    6763                 :         }
    6764                 : }
    6765                 : 
    6766                 : #else /* !defined MALLOC_DOUBLE_PURGE */
    6767                 : 
    6768                 : void
    6769               0 : jemalloc_purge_freed_pages()
    6770                 : {
    6771                 :         /* Do nothing. */
    6772               0 : }
    6773                 : 
    6774                 : #endif /* defined MALLOC_DOUBLE_PURGE */
    6775                 : 
    6776                 : 
    6777                 : 
    6778                 : #ifdef MOZ_MEMORY_WINDOWS
    6779                 : void*
    6780                 : _recalloc(void *ptr, size_t count, size_t size)
    6781                 : {
    6782                 :         size_t oldsize = (ptr != NULL) ? isalloc(ptr) : 0;
    6783                 :         size_t newsize = count * size;
    6784                 : 
    6785                 :         /*
    6786                 :          * In order for all trailing bytes to be zeroed, the caller needs to
    6787                 :          * use calloc(), followed by recalloc().  However, the current calloc()
    6788                 :          * implementation only zeros the bytes requested, so if recalloc() is
    6789                 :          * to work 100% correctly, calloc() will need to change to zero
    6790                 :          * trailing bytes.
    6791                 :          */
    6792                 : 
    6793                 :         ptr = realloc(ptr, newsize);
    6794                 :         if (ptr != NULL && oldsize < newsize) {
    6795                 :                 memset((void *)((uintptr_t)ptr + oldsize), 0, newsize -
    6796                 :                     oldsize);
    6797                 :         }
    6798                 : 
    6799                 :         return ptr;
    6800                 : }
    6801                 : 
    6802                 : /*
    6803                 :  * This impl of _expand doesn't ever actually expand or shrink blocks: it
    6804                 :  * simply replies that you may continue using a shrunk block.
    6805                 :  */
    6806                 : void*
    6807                 : _expand(void *ptr, size_t newsize)
    6808                 : {
    6809                 :         if (isalloc(ptr) >= newsize)
    6810                 :                 return ptr;
    6811                 : 
    6812                 :         return NULL;
    6813                 : }
    6814                 : 
    6815                 : size_t
    6816                 : _msize(const void *ptr)
    6817                 : {
    6818                 : 
    6819                 :         return malloc_usable_size(ptr);
    6820                 : }
    6821                 : #endif
    6822                 : 
    6823                 : /*
    6824                 :  * End non-standard functions.
    6825                 :  */
    6826                 : /******************************************************************************/
    6827                 : /*
    6828                 :  * Begin library-private functions, used by threading libraries for protection
    6829                 :  * of malloc during fork().  These functions are only called if the program is
    6830                 :  * running in threaded mode, so there is no need to check whether the program
    6831                 :  * is threaded here.
    6832                 :  */
    6833                 : 
    6834                 : static void
    6835               0 : _malloc_prefork(void)
    6836                 : {
    6837                 :         unsigned i;
    6838                 : 
    6839                 :         /* Acquire all mutexes in a safe order. */
    6840                 : 
    6841               0 :         malloc_spin_lock(&arenas_lock);
    6842               0 :         for (i = 0; i < narenas; i++) {
    6843               0 :                 if (arenas[i] != NULL)
    6844               0 :                         malloc_spin_lock(&arenas[i]->lock);
    6845                 :         }
    6846                 : 
    6847               0 :         malloc_mutex_lock(&base_mtx);
    6848                 : 
    6849               0 :         malloc_mutex_lock(&huge_mtx);
    6850               0 : }
    6851                 : 
    6852                 : static void
    6853               0 : _malloc_postfork(void)
    6854                 : {
    6855                 :         unsigned i;
    6856                 : 
    6857                 :         /* Release all mutexes, now that fork() has completed. */
    6858                 : 
    6859               0 :         malloc_mutex_unlock(&huge_mtx);
    6860                 : 
    6861               0 :         malloc_mutex_unlock(&base_mtx);
    6862                 : 
    6863               0 :         for (i = 0; i < narenas; i++) {
    6864               0 :                 if (arenas[i] != NULL)
    6865               0 :                         malloc_spin_unlock(&arenas[i]->lock);
    6866                 :         }
    6867               0 :         malloc_spin_unlock(&arenas_lock);
    6868               0 : }
    6869                 : 
    6870                 : /*
    6871                 :  * End library-private functions.
    6872                 :  */
    6873                 : /******************************************************************************/
    6874                 : 
    6875                 : #ifdef HAVE_DLOPEN
    6876                 : #  include <dlfcn.h>
    6877                 : #endif
    6878                 : 
    6879                 : #ifdef MOZ_MEMORY_DARWIN
    6880                 : 
    6881                 : static void *
    6882                 : zone_malloc(malloc_zone_t *zone, size_t size)
    6883                 : {
    6884                 : 
    6885                 :         return (malloc(size));
    6886                 : }
    6887                 : 
    6888                 : static void *
    6889                 : zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
    6890                 : {
    6891                 : 
    6892                 :         return (calloc(num, size));
    6893                 : }
    6894                 : 
    6895                 : static void *
    6896                 : zone_valloc(malloc_zone_t *zone, size_t size)
    6897                 : {
    6898                 :         void *ret = NULL; /* Assignment avoids useless compiler warning. */
    6899                 : 
    6900                 :         posix_memalign(&ret, pagesize, size);
    6901                 : 
    6902                 :         return (ret);
    6903                 : }
    6904                 : 
    6905                 : static void *
    6906                 : zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
    6907                 : {
    6908                 :         return (memalign(alignment, size));
    6909                 : }
    6910                 : 
    6911                 : static void *
    6912                 : zone_destroy(malloc_zone_t *zone)
    6913                 : {
    6914                 : 
    6915                 :         /* This function should never be called. */
    6916                 :         assert(false);
    6917                 :         return (NULL);
    6918                 : }
    6919                 : 
    6920                 : static size_t
    6921                 : zone_good_size(malloc_zone_t *zone, size_t size)
    6922                 : {
    6923                 :         return je_malloc_good_size(size);
    6924                 : }
    6925                 : 
    6926                 : static size_t
    6927                 : ozone_size(malloc_zone_t *zone, void *ptr)
    6928                 : {
    6929                 :         size_t ret = isalloc_validate(ptr);
    6930                 :         if (ret == 0)
    6931                 :                 ret = szone->size(zone, ptr);
    6932                 : 
    6933                 :         return ret;
    6934                 : }
    6935                 : 
    6936                 : static void
    6937                 : ozone_free(malloc_zone_t *zone, void *ptr)
    6938                 : {
    6939                 :         if (isalloc_validate(ptr) != 0)
    6940                 :                 free(ptr);
    6941                 :         else {
    6942                 :                 size_t size = szone->size(zone, ptr);
    6943                 :                 if (size != 0)
    6944                 :                         (szone->free)(zone, ptr);
    6945                 :                 /* Otherwise we leak. */
    6946                 :         }
    6947                 : }
    6948                 : 
    6949                 : static void *
    6950                 : ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
    6951                 : {
    6952                 :     size_t oldsize;
    6953                 :         if (ptr == NULL)
    6954                 :                 return (malloc(size));
    6955                 : 
    6956                 :         oldsize = isalloc_validate(ptr);
    6957                 :         if (oldsize != 0)
    6958                 :                 return (realloc(ptr, size));
    6959                 :         else {
    6960                 :                 oldsize = szone->size(zone, ptr);
    6961                 :                 if (oldsize == 0)
    6962                 :                         return (malloc(size));
    6963                 :                 else {
    6964                 :                         void *ret = malloc(size);
    6965                 :                         if (ret != NULL) {
    6966                 :                                 memcpy(ret, ptr, (oldsize < size) ? oldsize :
    6967                 :                                     size);
    6968                 :                                 (szone->free)(zone, ptr);
    6969                 :                         }
    6970                 :                         return (ret);
    6971                 :                 }
    6972                 :         }
    6973                 : }
    6974                 : 
    6975                 : static unsigned
    6976                 : ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results,
    6977                 :     unsigned num_requested)
    6978                 : {
    6979                 :         /* Don't bother implementing this interface, since it isn't required. */
    6980                 :         return 0;
    6981                 : }
    6982                 : 
    6983                 : static void
    6984                 : ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num)
    6985                 : {
    6986                 :         unsigned i;
    6987                 : 
    6988                 :         for (i = 0; i < num; i++)
    6989                 :                 ozone_free(zone, to_be_freed[i]);
    6990                 : }
    6991                 : 
    6992                 : static void
    6993                 : ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
    6994                 : {
    6995                 :         if (isalloc_validate(ptr) != 0) {
    6996                 :                 assert(isalloc_validate(ptr) == size);
    6997                 :                 free(ptr);
    6998                 :         } else {
    6999                 :                 assert(size == szone->size(zone, ptr));
    7000                 :                 l_szone.m16(zone, ptr, size);
    7001                 :         }
    7002                 : }
    7003                 : 
    7004                 : static void
    7005                 : ozone_force_lock(malloc_zone_t *zone)
    7006                 : {
    7007                 :         /* jemalloc locking is taken care of by the normal jemalloc zone. */
    7008                 :         szone->introspect->force_lock(zone);
    7009                 : }
    7010                 : 
    7011                 : static void
    7012                 : ozone_force_unlock(malloc_zone_t *zone)
    7013                 : {
    7014                 :         /* jemalloc locking is taken care of by the normal jemalloc zone. */
    7015                 :         szone->introspect->force_unlock(zone);
    7016                 : }
    7017                 : 
    7018                 : static size_t
    7019                 : zone_version_size(int version)
    7020                 : {
    7021                 :     switch (version)
    7022                 :     {
    7023                 :         case SNOW_LEOPARD_MALLOC_ZONE_T_VERSION:
    7024                 :             return sizeof(snow_leopard_malloc_zone);
    7025                 :         case LEOPARD_MALLOC_ZONE_T_VERSION:
    7026                 :             return sizeof(leopard_malloc_zone);
    7027                 :         default:
    7028                 :         case LION_MALLOC_ZONE_T_VERSION:
    7029                 :             return sizeof(lion_malloc_zone);
    7030                 :     }
    7031                 : }
    7032                 : 
    7033                 : /*
    7034                 :  * Overlay the default scalable zone (szone) such that existing allocations are
    7035                 :  * drained, and further allocations come from jemalloc. This is necessary
    7036                 :  * because Core Foundation directly accesses and uses the szone before the
    7037                 :  * jemalloc library is even loaded.
    7038                 :  */
    7039                 : static void
    7040                 : szone2ozone(malloc_zone_t *default_zone, size_t size)
    7041                 : {
    7042                 :     lion_malloc_zone *l_zone;
    7043                 :         assert(malloc_initialized);
    7044                 : 
    7045                 :         /*
    7046                 :          * Stash a copy of the original szone so that we can call its
    7047                 :          * functions as needed. Note that internally, the szone stores its
    7048                 :          * bookkeeping data structures immediately following the malloc_zone_t
    7049                 :          * header, so when calling szone functions, we need to pass a pointer to
    7050                 :          * the original zone structure.
    7051                 :          */
    7052                 :         memcpy(szone, default_zone, size);
    7053                 : 
    7054                 :         /* OSX 10.7 allocates the default zone in protected memory. */
    7055                 :         if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
    7056                 :                 void* start_of_page = (void*)((size_t)(default_zone) & ~pagesize_mask);
    7057                 :                 mprotect (start_of_page, size, PROT_READ | PROT_WRITE);
    7058                 :         }
    7059                 : 
    7060                 :         default_zone->size = (void *)ozone_size;
    7061                 :         default_zone->malloc = (void *)zone_malloc;
    7062                 :         default_zone->calloc = (void *)zone_calloc;
    7063                 :         default_zone->valloc = (void *)zone_valloc;
    7064                 :         default_zone->free = (void *)ozone_free;
    7065                 :         default_zone->realloc = (void *)ozone_realloc;
    7066                 :         default_zone->destroy = (void *)zone_destroy;
    7067                 :         default_zone->batch_malloc = NULL;
    7068                 :         default_zone->batch_free = ozone_batch_free;
    7069                 :         default_zone->introspect = ozone_introspect;
    7070                 : 
    7071                 :         /* Don't modify default_zone->zone_name; Mac libc may rely on the name
    7072                 :          * being unchanged.  See Mozilla bug 694896. */
    7073                 : 
    7074                 :         ozone_introspect->enumerator = NULL;
    7075                 :         ozone_introspect->good_size = (void *)zone_good_size;
    7076                 :         ozone_introspect->check = NULL;
    7077                 :         ozone_introspect->print = NULL;
    7078                 :         ozone_introspect->log = NULL;
    7079                 :         ozone_introspect->force_lock = (void *)ozone_force_lock;
    7080                 :         ozone_introspect->force_unlock = (void *)ozone_force_unlock;
    7081                 :         ozone_introspect->statistics = NULL;
    7082                 : 
    7083                 :     /* Platform-dependent structs */
    7084                 :     l_zone = (lion_malloc_zone*)(default_zone);
    7085                 : 
    7086                 :     if (default_zone->version >= SNOW_LEOPARD_MALLOC_ZONE_T_VERSION) {
    7087                 :         l_zone->m15 = (void (*)())zone_memalign;
    7088                 :         l_zone->m16 = (void (*)())ozone_free_definite_size;
    7089                 :         l_ozone_introspect.m9 = NULL;
    7090                 :     }
    7091                 : 
    7092                 :     if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
    7093                 :         l_zone->m17 = NULL;
    7094                 :         l_ozone_introspect.m10 = NULL;
    7095                 :         l_ozone_introspect.m11 = NULL;
    7096                 :         l_ozone_introspect.m12 = NULL;
    7097                 :         l_ozone_introspect.m13 = NULL;
    7098                 :     }
    7099                 : }
    7100                 : 
    7101                 : __attribute__((constructor))
    7102                 : void
    7103                 : jemalloc_darwin_init(void)
    7104                 : {
    7105                 :         if (malloc_init_hard())
    7106                 :                 abort();
    7107                 : }
    7108                 : 
    7109                 : #elif defined(__GLIBC__) && !defined(__UCLIBC__)
    7110                 : /*
    7111                 :  * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
    7112                 :  * to inconsistently reference libc's malloc(3)-compatible functions
    7113                 :  * (bug 493541).
    7114                 :  *
    7115                 :  * These definitions interpose hooks in glibc.  The functions are actually
    7116                 :  * passed an extra argument for the caller return address, which will be
    7117                 :  * ignored.
    7118                 :  */
    7119                 : void (*__free_hook)(void *ptr) = free;
    7120                 : void *(*__malloc_hook)(size_t size) = malloc;
    7121                 : void *(*__realloc_hook)(void *ptr, size_t size) = realloc;
    7122                 : void *(*__memalign_hook)(size_t alignment, size_t size) = MEMALIGN;
    7123                 : 
    7124                 : #elif defined(RTLD_DEEPBIND)
    7125                 : /*
    7126                 :  * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their
    7127                 :  * implementations permit similar inconsistencies?  Should STV_SINGLETON
    7128                 :  * visibility be used for interposition where available?
    7129                 :  */
    7130                 : #  error "Interposing malloc is unsafe on this system without libc malloc hooks."
    7131                 : #endif
    7132                 : 
    7133                 : #ifdef MOZ_MEMORY_WINDOWS
    7134                 : /*
    7135                 :  * In the new style jemalloc integration jemalloc is built as a separate
    7136                 :  * shared library.  Since we're no longer hooking into the CRT binary,
    7137                 :  * we need to initialize the heap at the first opportunity we get.
    7138                 :  * DLL_PROCESS_ATTACH in DllMain is that opportunity.
    7139                 :  */
    7140                 : BOOL APIENTRY DllMain(HINSTANCE hModule, 
    7141                 :                       DWORD reason, 
    7142                 :                       LPVOID lpReserved)
    7143                 : {
    7144                 :   switch (reason) {
    7145                 :     case DLL_PROCESS_ATTACH:
    7146                 :       /* Don't force the system to page DllMain back in every time
    7147                 :        * we create/destroy a thread */
    7148                 :       DisableThreadLibraryCalls(hModule);
    7149                 :       /* Initialize the heap */
    7150                 :       malloc_init_hard();
    7151                 :       break;
    7152                 :     
    7153                 :     case DLL_PROCESS_DETACH:
    7154                 :       break;
    7155                 : 
    7156                 :   }
    7157                 : 
    7158                 :   return TRUE;
    7159                 : }
    7160                 : 
    7161                 : /*
    7162                 :  *  There's a fun allocator mismatch in (at least) the VS 2010 CRT
    7163                 :  *  (see the giant comment in this directory's Makefile.in
    7164                 :  *  that gets redirected here to avoid a crash on shutdown.
    7165                 :  */
    7166                 : void
    7167                 : je_dumb_free_thunk(void *ptr)
    7168                 : {
    7169                 :   return; /* shutdown leaks that we don't care about */
    7170                 : }
    7171                 : 
    7172                 : #include <wchar.h>
    7173                 : 
    7174                 : /*
    7175                 :  *  We also need to provide our own impl of wcsdup so that we don't ask
    7176                 :  *  the CRT for memory from its heap (which will then be unfreeable).
    7177                 :  */
    7178                 : wchar_t *je_wcsdup(const wchar_t *src)
    7179                 : {
    7180                 :   size_t len = wcslen(src);
    7181                 :   wchar_t* dst = (wchar_t*)je_malloc((len + 1) * sizeof(wchar_t));
    7182                 :   if(dst)
    7183                 :     wcsncpy(dst, src, len + 1);
    7184                 :   return dst;
    7185                 : }
    7186                 : 
    7187                 : #endif

Generated by: LCOV version 1.7