Index: Zend/zend_alloc.c =================================================================== --- Zend/zend_alloc.c (revision 308156) +++ Zend/zend_alloc.c (working copy) @@ -59,6 +59,48 @@ # define PTR_FMT "0x%0.8lx" #endif +#if 0 +# define ZEND_MM_PCACHE_ENABLED +#endif + +#ifdef ZEND_MM_PCACHE_ENABLED +# define ZEND_MM_PCACHE_ALIGN_SIZE 4096 +# define ZEND_MM_PCACHE_MAX_SIZE 256 +# define ZEND_MM_PCACHE_THRESHOLD_COUNT 1 +# define ZEND_MM_PCACHE_LIMIT 12582912 +# define ZEND_MM_PCACHE_IGNORE_CNT 10 + +typedef struct _zend_mm_mem_pool { + size_t size; + size_t free; + void *data; + void *data_end; +} zend_mm_mem_pool; + +typedef struct _zend_mm_pcache { + size_t stats[ZEND_MM_PCACHE_MAX_SIZE]; + size_t stats_sum[ZEND_MM_PCACHE_MAX_SIZE]; + zend_mm_mem_pool pools[ZEND_MM_PCACHE_MAX_SIZE]; + int cnt; +} zend_mm_pcache; +#endif + +typedef struct _zend_alloc_globals { + zend_mm_heap *mm_heap; +#ifdef ZEND_MM_PCACHE_ENABLED + zend_mm_pcache *pcache; +#endif +} zend_alloc_globals; + +#ifdef ZTS +static int alloc_globals_id; +# define AG(v) TSRMG(alloc_globals_id, zend_alloc_globals *, v) +#else +# define AG(v) (alloc_globals.v) +static zend_alloc_globals alloc_globals; +#endif + + #if ZEND_DEBUG void zend_debug_alloc_output(char *format, ...) { @@ -715,6 +757,156 @@ #endif } +#ifdef ZEND_MM_PCACHE_ENABLED +zend_mm_pcache *zend_mm_pcache_init(void) /* {{{ */ +{ + zend_mm_pcache *pcache; + + pcache = calloc(1, sizeof(zend_mm_pcache)); + +#if 0 + for (i = ZEND_MM_ALIGNED_HEADER_SIZE; i < ZEND_MM_PCACHE_MAX_SIZE; i++) { + pool = &pcache->pools[i]; + pool->size = 1024; + pool->data = malloc(i * pool->size); + if (!pool->data) { + pool->size = 0; + pool->data_end = NULL; + } else { + pool->data_end = pool->data + i * pool->size; + } + pool->free = pool->size; + } +#endif + return pcache; +} +/* }}} */ + +#if 0 +# define ZEND_MM_PCACHE_STATS +#endif +void zend_mm_pcache_rebuild(zend_mm_pcache *pcache) /* {{{ */ +{ + int i; + zend_mm_mem_pool *pool; + size_t overall_size = 0; + +#ifdef ZEND_MM_PCACHE_STATS + FILE *fp; + pid_t pid = getpid(); + char buf[512]; + int len; +#endif + + /* gather some stats during the first requests */ + if (pcache->cnt < ZEND_MM_PCACHE_IGNORE_CNT) { + for (i = ZEND_MM_ALIGNED_HEADER_SIZE; i < ZEND_MM_PCACHE_MAX_SIZE; i++) { + pcache->stats_sum[i] += pcache->stats[i]; + } + pcache->cnt++; + return; + } + +#ifdef ZEND_MM_PCACHE_STATS + if (pcache->cnt < (ZEND_MM_PCACHE_IGNORE_CNT + 10)) { + pcache->cnt++; + sprintf(buf, "/tmp/fpm_%d.memstats.txt", pid); + fp = fopen(buf, "a+"); + + if (fp) { + fwrite("-------\n", sizeof("-------\n")-1, 1, fp); + } + + for (i = ZEND_MM_ALIGNED_HEADER_SIZE; i < ZEND_MM_PCACHE_MAX_SIZE; i++) { + pool = &pcache->pools[i]; + + overall_size += i * pool->size; + + if (fp && (pcache->stats[i] > ZEND_MM_PCACHE_THRESHOLD_COUNT || pool->size)) { + len = sprintf(buf, "%d bytes - %zu (%zu preallocated blocks, %zu free)\n", i, pcache->stats[i], pool->size, pool->free); + fwrite(buf, len, 1, fp); + } + } + + if (fp) { + len = sprintf(buf, "cache size: %zu Mb\n", overall_size/(1024*1024)); + fwrite(buf, len, 1, fp); + fclose(fp); + } + } +#endif + + if (pcache->cnt == ZEND_MM_PCACHE_IGNORE_CNT) { + int last_one = 0; + + pcache->cnt++; + overall_size = 0; + + for (i = ZEND_MM_ALIGNED_HEADER_SIZE; i < ZEND_MM_PCACHE_MAX_SIZE; i++) { + size_t avg_size = pcache->stats_sum[i] / ZEND_MM_PCACHE_IGNORE_CNT; + size_t pool_size; + + pool = &pcache->pools[i]; + + if (avg_size < ZEND_MM_PCACHE_THRESHOLD_COUNT) { + /* don't bother caching small amounts */ + continue; + } + + pool_size = i * (((avg_size / ZEND_MM_PCACHE_ALIGN_SIZE) + 1) * ZEND_MM_PCACHE_ALIGN_SIZE); + if ((overall_size + pool_size) > ZEND_MM_PCACHE_LIMIT) { + pool_size = ZEND_MM_PCACHE_LIMIT - overall_size; + pool_size = (pool_size / i) * i; + last_one = 1; + } + + overall_size += pool_size; + + pool->size = pool_size / i; + pool->data = malloc(pool_size); + if (!pool->data) { + pool->size = 0; + pool->data_end = NULL; + } else { + pool->data_end = pool->data + i * pool->size; + } + pool->free = pool->size; + if (last_one) { + break; + } + } + } else { + for (i = ZEND_MM_ALIGNED_HEADER_SIZE; i < ZEND_MM_PCACHE_MAX_SIZE; i++) { + pool = &pcache->pools[i]; + pool->free = pool->size; + } + } +} +/* }}} */ + +void zend_mm_pcache_request_shutdown(zend_mm_pcache *pcache) /* {{{ */ +{ + zend_mm_pcache_rebuild(pcache); + memset(pcache->stats, 0, sizeof(pcache->stats)); +} +/* }}} */ + +void zend_mm_pcache_shutdown(zend_mm_pcache *pcache) /* {{{ */ +{ + int i; + zend_mm_mem_pool *pool; + + for (i = ZEND_MM_ALIGNED_HEADER_SIZE; i < ZEND_MM_PCACHE_MAX_SIZE; i++) { + pool = &(pcache->pools[i]); + if (pool->data) { + free(pool->data); + } + } + free(pcache); +} +/* }}} */ +#endif + static inline void zend_mm_add_to_rest_list(zend_mm_heap *heap, zend_mm_free_block *mm_block) { zend_mm_free_block *prev, *next; @@ -1652,6 +1844,9 @@ if (!internal) { free(heap); } +#ifdef ZEND_MM_PCACHE_ENABLED + zend_mm_pcache_shutdown(AG(pcache)); +#endif } else { if (heap->compact_size && heap->real_peak > heap->compact_size) { @@ -1667,6 +1862,9 @@ heap->reserve = _zend_mm_alloc_int(heap, heap->reserve_size ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC); } heap->overflow = 0; +#ifdef ZEND_MM_PCACHE_ENABLED + zend_mm_pcache_request_shutdown(AG(pcache)); +#endif } } @@ -1817,6 +2015,37 @@ zend_mm_segment *segment; int keep_rest = 0; +#ifdef ZEND_MM_PCACHE_ENABLED + if (EXPECTED(true_size < ZEND_MM_PCACHE_MAX_SIZE)) { + zend_mm_mem_pool *pool; + + AG(pcache)->stats[true_size]++; + pool = &AG(pcache)->pools[true_size]; + + /* one block at the end must be left free (ZEND_MM_BLOCK writes to it) */ + if (pool->free > 1) { + + if (pool->free == pool->size) { + best_fit = (zend_mm_free_block *) ((char *) pool->data); + ZEND_MM_MARK_FIRST_BLOCK(best_fit); + block_size = pool->size * true_size; + ZEND_MM_LAST_BLOCK(ZEND_MM_BLOCK_AT(best_fit, block_size - true_size)); + } else { + best_fit = (zend_mm_free_block *) (((char *) pool->data + (pool->size - pool->free) * true_size)); + } + pool->free--; + + /* this is enough to make the block pass all the checks */ + ZEND_MM_BLOCK(best_fit, ZEND_MM_USED_BLOCK, true_size); + + ZEND_MM_SET_DEBUG_INFO(best_fit, size, 1, 0); + return ZEND_MM_DATA_OF(best_fit); + } + } +#endif + + //fprintf(stderr, "allocating %zu bytes\n", true_size); + if (EXPECTED(ZEND_MM_SMALL_SIZE(true_size))) { size_t index = ZEND_MM_BUCKET_INDEX(true_size); size_t bitmap; @@ -1998,6 +2227,15 @@ size = ZEND_MM_BLOCK_SIZE(mm_block); ZEND_MM_CHECK_PROTECTION(mm_block); +#ifdef ZEND_MM_PCACHE_ENABLED + if (size < ZEND_MM_PCACHE_MAX_SIZE) { + zend_mm_mem_pool *pool = &AG(pcache)->pools[size]; + if (pool->data && pool->data <= p && pool->data_end > p) { + return; + } + } +#endif + #if ZEND_DEBUG || ZEND_MM_HEAP_PROTECTION memset(ZEND_MM_DATA_OF(mm_block), 0x5a, mm_block->debug.size); #endif @@ -2055,11 +2293,35 @@ if (UNEXPECTED(!p) || !ZEND_MM_VALID_PTR(p)) { return _zend_mm_alloc_int(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); } + mm_block = ZEND_MM_HEADER_OF(p); true_size = ZEND_MM_TRUE_SIZE(size); orig_size = ZEND_MM_BLOCK_SIZE(mm_block); ZEND_MM_CHECK_PROTECTION(mm_block); +#ifdef ZEND_MM_PCACHE_ENABLED + if (orig_size < ZEND_MM_PCACHE_MAX_SIZE) { + zend_mm_mem_pool *pool = &AG(pcache)->pools[orig_size]; + if (pool->data && pool->data <= p && pool->data_end > p) { + ptr = _zend_mm_alloc_int(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + if (orig_size <= true_size) { +#if ZEND_DEBUG || ZEND_MM_HEAP_PROTECTION + memcpy(ptr, p, mm_block->debug.size); +#else + memcpy(ptr, p, orig_size - ZEND_MM_ALIGNED_HEADER_SIZE); +#endif + } else { +#if ZEND_DEBUG || ZEND_MM_HEAP_PROTECTION + memcpy(ptr, p, size); +#else + memcpy(ptr, p, true_size - ZEND_MM_ALIGNED_HEADER_SIZE); +#endif + } + return ptr; + } + } +#endif + if (UNEXPECTED(true_size < size)) { goto out_of_memory; } @@ -2323,18 +2585,6 @@ /* Allocation Manager */ /**********************/ -typedef struct _zend_alloc_globals { - zend_mm_heap *mm_heap; -} zend_alloc_globals; - -#ifdef ZTS -static int alloc_globals_id; -# define AG(v) TSRMG(alloc_globals_id, zend_alloc_globals *, v) -#else -# define AG(v) (alloc_globals.v) -static zend_alloc_globals alloc_globals; -#endif - ZEND_API int is_zend_mm(TSRMLS_D) { return AG(mm_heap)->use_zend_alloc; @@ -2565,7 +2815,11 @@ static void alloc_globals_ctor(zend_alloc_globals *alloc_globals TSRMLS_DC) { char *tmp; + alloc_globals->mm_heap = zend_mm_startup(); +#ifdef ZEND_MM_PCACHE_ENABLED + alloc_globals->pcache = zend_mm_pcache_init(); +#endif tmp = getenv("USE_ZEND_ALLOC"); if (tmp) {